From 2a3ce332862400d301076f80373b56e06a8b30f3 Mon Sep 17 00:00:00 2001 From: William Durand Date: Fri, 18 Dec 2020 12:13:48 +0100 Subject: [PATCH] Migrate to Circle CI (#16189) --- .circleci/config.yml | 467 +++++++++++++++++- .travis.yml | 109 ---- README.rst | 4 +- contribute.json | 4 +- docker-compose.yml | 2 +- requirements/{travis_base.txt => ci_base.txt} | 0 ...ig.yaml => autograph_localdev_config.yaml} | 5 +- src/olympia/amo/search.py | 5 - src/olympia/lib/settings_base.py | 2 +- .../reviewers/tests/test_serializers.py | 2 +- src/olympia/versions/tests/test_utils.py | 2 +- 11 files changed, 467 insertions(+), 135 deletions(-) delete mode 100644 .travis.yml rename requirements/{travis_base.txt => ci_base.txt} (100%) rename scripts/{autograph_travis_test_config.yaml => autograph_localdev_config.yaml} (98%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 06e6b50288..9ded751327 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,321 @@ +# These environment variables must be set in CircleCI UI +# +# DOCKERHUB_REPO - docker hub repo, format: / +# DOCKER_USER - Login user for docker hub +# DOCKER_PASS - Login password for docker hub user version: 2.1 +orbs: + # This is needed to be able to install Node.js in the primary container, + # which provides python only. + node: circleci/node@4.1.0 + +references: + # We declare the autograph configuration here to be able to fully leverage + # Docker executors. This configuration should be kept in sync with the + # content of `scripts/autograph_localdev_config.yaml`, which is used for + # local dev. Sadly, we cannot "include" this file here. + autograph_config: &autograph_config | + # Note: Most of the configuration here got copied from + # https://github.com/mozilla-services/autograph/blob/master/autograph.yaml + server: + # This port should be perfectly free, the upstream default of 8000 is + # used by django sometimes so let's not do that. + listen: "0.0.0.0:5500" + # cache 500k nonces to protect from authorization replay attacks + noncecachesize: 10 + + # The keys below are testing keys that do not grant any power + signers: + - id: webextensions-rsa + type: xpi + # The signing parameters for each type of add-on are 'add-on' are + # signed with the OU 'Production' and the provided ID 'extension' are + # signed with the OU 'Mozilla Extensions' and the provided ID 'system + # add-on' are signed with the OU 'Mozilla Components' and the + # provided ID + mode: add-on + recommendation: + path: "mozilla-recommendation.json" + certificate: | + -----BEGIN CERTIFICATE----- + MIIH0zCCBbugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBvDELMAkGA1UEBhMCVVMx + CzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRwwGgYDVQQKExNB + bGxpem9tIENvcnBvcmF0aW9uMSAwHgYDVQQLExdBbGxpem9tIEFNTyBEZXZlbG9w + bWVudDEYMBYGA1UEAxMPZGV2LmFtby5yb290LmNhMS4wLAYJKoZIhvcNAQkBFh9m + b3hzZWMrZGV2YW1vcm9vdGNhQG1vemlsbGEuY29tMB4XDTE3MDMyMTIzNDQwNFoX + DTI3MDMxOTIzNDQwNFowgbwxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQG + A1UEBxMNTW91bnRhaW4gVmlldzEcMBoGA1UEChMTQWxsaXpvbSBDb3Jwb3JhdGlv + bjEgMB4GA1UECxMXQWxsaXpvbSBBTU8gRGV2ZWxvcG1lbnQxGDAWBgNVBAMTD2Rl + di5hbW8ucm9vdC5jYTEuMCwGCSqGSIb3DQEJARYfZm94c2VjK2RldmFtb3Jvb3Rj + YUBtb3ppbGxhLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMdX + 5soUuvWnkVHRHN5BKByrgpuU3QioE8SNT7BwRFeqbOySdvu5ecQAdNUoRbRyFmNB + ety2rQM9qw6y8eSe9fufIgrv1sg/xj7vweLmuC8Ob+zo5/iwRQw4JUdXnDjwX3W0 + auh0QRYfxWGK3hVrP9j1zIJk/yRBornCvXTtn8C/hVSE/PWc6CuV8vTcpyj+TPni + Lvulq17NdlX5qgUdn1yougJxnznkwnoIaBYLdAyZJJIUEomiEIxfabjnh8rfSMIw + AqmslrC8F73yo4JrCqJPt1ipggfpO3ZAjlEoTMcTUgyqR8B35GyuywWR0XrkJV7N + A7BM1qNjLb2to0XQSrGyWA7uPw88LuVk2aUPDE5uNK5Kv//+SGChUn2fDZTsjj3J + KY7f39JVwh/nk8ZkApplne8fKPoknW7er2R+rejyBx1+fJjLegKQsATpgKz4LRf4 + ct34oWSV6QXrZ/KKW+frWoHncy8C+UnCC3cDBKs272yqOvBoGMQTrF5oMn8i/Rap + gBbBdwysdJXb+buf/+ZS0PUt7avKFIlXqCNZjG3xotBsTuCL5zAoVKoXJW1FwrcZ + pveQuishKWNf9Id+0HaBdDp/vlbrTwXD1zsxfYvYw8wI7NkNO3TQBni5iyG4B1wh + oR+Z5AebWuJqVnsJyjPakNiuhKNsO/xTa4TF/ymfAgMBAAGjggHcMIIB2DAPBgNV + HRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAWBgNVHSUBAf8EDDAKBggrBgEF + BQcDAzAdBgNVHQ4EFgQU2LRpqTdeQ1QlBWNA6fYAqHdpSaUwgekGA1UdIwSB4TCB + 3oAU2LRpqTdeQ1QlBWNA6fYAqHdpSaWhgcKkgb8wgbwxCzAJBgNVBAYTAlVTMQsw + CQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEcMBoGA1UEChMTQWxs + aXpvbSBDb3Jwb3JhdGlvbjEgMB4GA1UECxMXQWxsaXpvbSBBTU8gRGV2ZWxvcG1l + bnQxGDAWBgNVBAMTD2Rldi5hbW8ucm9vdC5jYTEuMCwGCSqGSIb3DQEJARYfZm94 + c2VjK2RldmFtb3Jvb3RjYUBtb3ppbGxhLmNvbYIBATBCBglghkgBhvhCAQQENRYz + aHR0cHM6Ly9jb250ZW50LXNpZ25hdHVyZS5kZXYubW96YXdzLm5ldC9jYS9jcmwu + cGVtME4GCCsGAQUFBwEBBEIwQDA+BggrBgEFBQcwAoYyaHR0cHM6Ly9jb250ZW50 + LXNpZ25hdHVyZS5kZXYubW96YXdzLm5ldC9jYS9jYS5wZW0wDQYJKoZIhvcNAQEL + BQADggIBALqVt54WTkxD5U5fHPRUSZA9rFigoIcrHNrq+gTDd057cBDUWNc0cEHV + qaP0zgzqD2bIhV/WWlfMDY3VnB8L2+Vjvu2CEt8/9Kh5x9IgBmZt5VUMuEdmQOyH + vA7lz3UI+jmUGcojtLsi+sf4kxDZh3QB3T/wGiHg+K7vXnY7GWEy1Cjfwk/dvbT2 + ODTb5B3SPGsh75VmfzFGgerzsS71LN4FYBRUklLe8ozqKF8r/jGE2vfDR1Cy09pN + oR9ti+zaBiEtMlWJjxYrv7HvuoDR9xLmPxyV6gQbo6NnbudkpNdg5LhbY3WV1IgL + TnwJ7aHXgzOZ3w/VsSctg4beZZgYnr81vLKyefWJH1VzCe5XTgwXC1R/afGiVJ0P + hA1+T4My9oTaQBsiNYA2keXKJbTKerMTupoLgV/lJjxfF5BfQiy9NL18/bzxqf+J + 7w4P/4oHt3QCdISAIhlG4ttXfRR8oz6obAb6QYdCf3x9b2/3UXKd3UJ+gwchPjj6 + InnLK8ig9scn4opVNkBkjlMRsq1yd017eQzLSirpKj3br69qyLoyb/nPNJi7bL1K + bf6m5mF5GmKR+Glvq74O8rLQZ3a75v6H+NwOqAlZnWSJmC84R2HHsHPBw+2pExJT + E5bRcttRlhEdN4NJ2vWJnOH0DENHy6TEwACINJVx6ftucfPfvOxI + -----END CERTIFICATE----- + privatekey: | + -----BEGIN PRIVATE KEY----- + MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDHV+bKFLr1p5FR + 0RzeQSgcq4KblN0IqBPEjU+wcERXqmzsknb7uXnEAHTVKEW0chZjQXrctq0DPasO + svHknvX7nyIK79bIP8Y+78Hi5rgvDm/s6Of4sEUMOCVHV5w48F91tGrodEEWH8Vh + it4Vaz/Y9cyCZP8kQaK5wr107Z/Av4VUhPz1nOgrlfL03Kco/kz54i77patezXZV + +aoFHZ9cqLoCcZ855MJ6CGgWC3QMmSSSFBKJohCMX2m454fK30jCMAKprJawvBe9 + 8qOCawqiT7dYqYIH6Tt2QI5RKEzHE1IMqkfAd+RsrssFkdF65CVezQOwTNajYy29 + raNF0EqxslgO7j8PPC7lZNmlDwxObjSuSr///khgoVJ9nw2U7I49ySmO39/SVcIf + 55PGZAKaZZ3vHyj6JJ1u3q9kfq3o8gcdfnyYy3oCkLAE6YCs+C0X+HLd+KFklekF + 62fyilvn61qB53MvAvlJwgt3AwSrNu9sqjrwaBjEE6xeaDJ/Iv0WqYAWwXcMrHSV + 2/m7n//mUtD1Le2ryhSJV6gjWYxt8aLQbE7gi+cwKFSqFyVtRcK3Gab3kLorISlj + X/SHftB2gXQ6f75W608Fw9c7MX2L2MPMCOzZDTt00AZ4uYshuAdcIaEfmeQHm1ri + alZ7Ccoz2pDYroSjbDv8U2uExf8pnwIDAQABAoICADf7eqgD3GGC1q/Yfzf3qnEq + xXo1+0EkGrEXUmrljHvmM8LYeyvEcerWifkW30SGybzENeHoN3xyhCiTnpUrAz/P + 9/qEUphYOK+SG6xCSTWF427wFb1km2+MEQQRGaFv+A8RRPjVNTYmZAM5wZbYUMz4 + cp+oB3NCL5Xll9lPpo61+pa65mN/1j/vU5TqptM/X5TJrZIke5UbNIF+pP3czNVz + 2RE4oZPbp7YnyDtwqf2jwH55vp8CcY1KemFgPGWAAWnvm7/U5Vjq6ewBSWQl9Y2R + v5bZu9fG61kRViZ6n91EksVVyOLHiNHw4LlGs0LE8a3G+6M2YQzvnHfpXLINhfwU + SZ6BWAJdknVsu6eesYoC08+nyikkq/A3BVD65pT5C9VsmUPbqqpGSYZmAuFgsf9m + zdyKVH4fOPx82DqSZEHZBojg3s5K141DmPp6o0OBX8Ydgfkg2sWXuNi/noBDvh9O + FXWN2IcgK0dET3pX4xFei0QuZgglDp3VyVVSCSUPsOwecZ2XTjtBZPCQVpp3r+QV + LyecFudQ94Ki/0R+M4CrE/mPApDvq+pTjYKFZ10YWtGIdguXq5BVZIMZfZzwIPWN + HdoaFnXRTXTlR4pLIM2nlOvyZmSMo0x6nzUMVGdv4Km9pxi6ZKAgAt4DkbCF9mt0 + QG8RpGJhiIch4kgKFmqxAoIBAQDw4X9Fp9t4f2UiessUDYxLyAtq4acu4ahup5Eb + vlDZPf9gInvz5q9aFHtYgtjTlH449f+EB4isKQscVMysgrJK+7z1IXXMm0sg44wT + F4oV+kvg3KpAridRHyE456RvCPqXYzty6ywJ9B7Zf2oCvd40JUOTm8z11K4COLut + rFIW/24PJA1CWudY/EgzD164k6379On0KryA77iKEZMUztBfHm/bdO8J/zmp7g+E + FS2TCBzR4LpN0uhBwp9wh4rVr74LrPDnQJVZKgeFd24UHEtmcVprAFNUexb2yy1s + vxnHsRPmv5eF7ED1Wlz2K+7LUWqibYOrjeCrS85+CEcey0ApAoIBAQDT2vmbHosb + Qr6ZENt6UX6n0RF8i4g3G4qhucr5hEMQs4H2J8SrUM68QT0GVY0GoDW6f79Pcyr0 + W1tm7qbAOm1Iv4uNYVL1qgpq1GnD5qpWSACGsVSE3OGELlNaVz8fgVdz6zT+rU2A + tp2t795UlrvaLgFI4wITqJF3LoTfy2MZu8JYCzlKM5pZksmEmJfR0RDAot2grtD3 + H5A+PZfUIZ/8BhmdaOAv5i647unfVF6UpPYejZ0rb67oEazxdeIHK3aD5AjurdsO + UpW/PMwsbaltp4iI7hvUfRX7Afb5fPXIhv9pHh1xWYl3djUNWmFoiMMP4tuxpOBo + y+T4maQaiDSHAoIBADrlZ9EIMclMnNXJYE4O4fbFesUvV0lHM3+ayQgXiH0Vg5Nl + 2xjPlqBX0bDajVluPU6AF3GYxfoSLv1GXqTvb9iVpKXrAHp+nef0uxMP9ltZT6Qz + UA1wh3x2OBFJ0hK0B1FsmeSHS8VDQye615jEA8iMM/GrbnnM/p7ccEcOkyO8YJSj + I/rNbzN6u8yAPZCzyx6Hy4w/xsdf1acslOHJj3kyX/cwqCGxnc/GvVR2OSZyHVnT + sLnGj7NEeudwvKlyxuzj5CMmz111wVEI2olgQa9Sl+EBu140mnDNTNYCA7OnwE3z + GoFMOrXC2mf2ZfSge4orbL5Nellnt51pOLp2x8ECggEBALM8Mazw/FOF9mbdgjJM + PFGSaa7rBcVJwdHttDHBmlPI6wzsvFEMPru6nfx76KJQbORqK9r13sN5fyzof59m + TwsbMt/cFSnOQJ39M7YPstDofbl20cDOduUzpEVsRvVKokhqGB3XVRiuZ1y+8WSz + Wh7OiTu3AwzKsrcYXkZQdnlRBq0iYcfLPKzHqUJLLzbOH9Q6djL5c8V/qLNfvNI1 + 2HqKVqV8Ex+zKJhBWRAe+x3bKnbS7MPQ6zNfsOdgCmhydwRCquPzpr7JU/PFZh+4 + b31cHgFrIZR2d2AzW1XcSLzsqa2vUs2RKOIu2deAPaUI/66zCZeTnGBNEFza76Ga + 1oUCggEAA38oXcnputwL103SeD8+uwHjtTf183Rucr+Ryqz6GymiWjlzELqu7TRd + yadAaNg9CuXmYS33Jtk/UNS0k9FvYqGTR+SBXIZr6nt9ZFd0SNlQkwkAQCsuekEs + nJlmUZax7DxXMgIHMKDboHZYM/MhgzEGSALmhU5LZ76MS17v3NEPxYpVHxjAotxW + g03HjWTltS8Bgt6u0KFTGJKEUcfwvWKZtjk5Fc1heZ49zh1nU3zo9C/h8iiijTy2 + s/YksP6cxveae4b7soN4rD/vnfsmKcG+DnTf6B8Zbm6tI2TneYOfFSCryp+yDnaJ + PIDNiTxNecePOmrD+1ivAEXcoL+e1w== + -----END PRIVATE KEY----- + + - id: webextensions-rsa-with-recommendation + type: xpi + # The signing parameters for each type of add-on are 'add-on' are + # signed with the OU 'Production' and the provided ID + # 'add-on-with-recommendation' are signed with the OU 'Production' + # and the provided ID and add a recommendation file 'extension' are + # signed with the OU 'Mozilla Extensions' and the provided ID 'system + # add-on' are signed with the OU 'Mozilla Components' and the + # provided ID 'hotfix' are signed with the OU 'Production' and the ID + # 'firefox-hotfix@mozilla.org' + mode: add-on-with-recommendation + recommendation: + path: "mozilla-recommendation.json" + states: + recommended: true + recommended-android: true + verified: true + line: true + relative_start: 0h + duration: 26298h + # RSA key gen is slow and CPU intensive, so we can optionally + # pregenerate and cache keys with a worker pool + rsacacheconfig: + numkeys: 25 + numgenerators: 2 + generatorsleepduration: 1m + fetchtimeout: 100ms + statssamplerate: 1m + certificate: | + -----BEGIN CERTIFICATE----- + MIIH0zCCBbugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBvDELMAkGA1UEBhMCVVMx + CzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRwwGgYDVQQKExNB + bGxpem9tIENvcnBvcmF0aW9uMSAwHgYDVQQLExdBbGxpem9tIEFNTyBEZXZlbG9w + bWVudDEYMBYGA1UEAxMPZGV2LmFtby5yb290LmNhMS4wLAYJKoZIhvcNAQkBFh9m + b3hzZWMrZGV2YW1vcm9vdGNhQG1vemlsbGEuY29tMB4XDTE3MDMyMTIzNDQwNFoX + DTI3MDMxOTIzNDQwNFowgbwxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQG + A1UEBxMNTW91bnRhaW4gVmlldzEcMBoGA1UEChMTQWxsaXpvbSBDb3Jwb3JhdGlv + bjEgMB4GA1UECxMXQWxsaXpvbSBBTU8gRGV2ZWxvcG1lbnQxGDAWBgNVBAMTD2Rl + di5hbW8ucm9vdC5jYTEuMCwGCSqGSIb3DQEJARYfZm94c2VjK2RldmFtb3Jvb3Rj + YUBtb3ppbGxhLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMdX + 5soUuvWnkVHRHN5BKByrgpuU3QioE8SNT7BwRFeqbOySdvu5ecQAdNUoRbRyFmNB + ety2rQM9qw6y8eSe9fufIgrv1sg/xj7vweLmuC8Ob+zo5/iwRQw4JUdXnDjwX3W0 + auh0QRYfxWGK3hVrP9j1zIJk/yRBornCvXTtn8C/hVSE/PWc6CuV8vTcpyj+TPni + Lvulq17NdlX5qgUdn1yougJxnznkwnoIaBYLdAyZJJIUEomiEIxfabjnh8rfSMIw + AqmslrC8F73yo4JrCqJPt1ipggfpO3ZAjlEoTMcTUgyqR8B35GyuywWR0XrkJV7N + A7BM1qNjLb2to0XQSrGyWA7uPw88LuVk2aUPDE5uNK5Kv//+SGChUn2fDZTsjj3J + KY7f39JVwh/nk8ZkApplne8fKPoknW7er2R+rejyBx1+fJjLegKQsATpgKz4LRf4 + ct34oWSV6QXrZ/KKW+frWoHncy8C+UnCC3cDBKs272yqOvBoGMQTrF5oMn8i/Rap + gBbBdwysdJXb+buf/+ZS0PUt7avKFIlXqCNZjG3xotBsTuCL5zAoVKoXJW1FwrcZ + pveQuishKWNf9Id+0HaBdDp/vlbrTwXD1zsxfYvYw8wI7NkNO3TQBni5iyG4B1wh + oR+Z5AebWuJqVnsJyjPakNiuhKNsO/xTa4TF/ymfAgMBAAGjggHcMIIB2DAPBgNV + HRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAWBgNVHSUBAf8EDDAKBggrBgEF + BQcDAzAdBgNVHQ4EFgQU2LRpqTdeQ1QlBWNA6fYAqHdpSaUwgekGA1UdIwSB4TCB + 3oAU2LRpqTdeQ1QlBWNA6fYAqHdpSaWhgcKkgb8wgbwxCzAJBgNVBAYTAlVTMQsw + CQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEcMBoGA1UEChMTQWxs + aXpvbSBDb3Jwb3JhdGlvbjEgMB4GA1UECxMXQWxsaXpvbSBBTU8gRGV2ZWxvcG1l + bnQxGDAWBgNVBAMTD2Rldi5hbW8ucm9vdC5jYTEuMCwGCSqGSIb3DQEJARYfZm94 + c2VjK2RldmFtb3Jvb3RjYUBtb3ppbGxhLmNvbYIBATBCBglghkgBhvhCAQQENRYz + aHR0cHM6Ly9jb250ZW50LXNpZ25hdHVyZS5kZXYubW96YXdzLm5ldC9jYS9jcmwu + cGVtME4GCCsGAQUFBwEBBEIwQDA+BggrBgEFBQcwAoYyaHR0cHM6Ly9jb250ZW50 + LXNpZ25hdHVyZS5kZXYubW96YXdzLm5ldC9jYS9jYS5wZW0wDQYJKoZIhvcNAQEL + BQADggIBALqVt54WTkxD5U5fHPRUSZA9rFigoIcrHNrq+gTDd057cBDUWNc0cEHV + qaP0zgzqD2bIhV/WWlfMDY3VnB8L2+Vjvu2CEt8/9Kh5x9IgBmZt5VUMuEdmQOyH + vA7lz3UI+jmUGcojtLsi+sf4kxDZh3QB3T/wGiHg+K7vXnY7GWEy1Cjfwk/dvbT2 + ODTb5B3SPGsh75VmfzFGgerzsS71LN4FYBRUklLe8ozqKF8r/jGE2vfDR1Cy09pN + oR9ti+zaBiEtMlWJjxYrv7HvuoDR9xLmPxyV6gQbo6NnbudkpNdg5LhbY3WV1IgL + TnwJ7aHXgzOZ3w/VsSctg4beZZgYnr81vLKyefWJH1VzCe5XTgwXC1R/afGiVJ0P + hA1+T4My9oTaQBsiNYA2keXKJbTKerMTupoLgV/lJjxfF5BfQiy9NL18/bzxqf+J + 7w4P/4oHt3QCdISAIhlG4ttXfRR8oz6obAb6QYdCf3x9b2/3UXKd3UJ+gwchPjj6 + InnLK8ig9scn4opVNkBkjlMRsq1yd017eQzLSirpKj3br69qyLoyb/nPNJi7bL1K + bf6m5mF5GmKR+Glvq74O8rLQZ3a75v6H+NwOqAlZnWSJmC84R2HHsHPBw+2pExJT + E5bRcttRlhEdN4NJ2vWJnOH0DENHy6TEwACINJVx6ftucfPfvOxI + -----END CERTIFICATE----- + privatekey: | + -----BEGIN PRIVATE KEY----- + MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDHV+bKFLr1p5FR + 0RzeQSgcq4KblN0IqBPEjU+wcERXqmzsknb7uXnEAHTVKEW0chZjQXrctq0DPasO + svHknvX7nyIK79bIP8Y+78Hi5rgvDm/s6Of4sEUMOCVHV5w48F91tGrodEEWH8Vh + it4Vaz/Y9cyCZP8kQaK5wr107Z/Av4VUhPz1nOgrlfL03Kco/kz54i77patezXZV + +aoFHZ9cqLoCcZ855MJ6CGgWC3QMmSSSFBKJohCMX2m454fK30jCMAKprJawvBe9 + 8qOCawqiT7dYqYIH6Tt2QI5RKEzHE1IMqkfAd+RsrssFkdF65CVezQOwTNajYy29 + raNF0EqxslgO7j8PPC7lZNmlDwxObjSuSr///khgoVJ9nw2U7I49ySmO39/SVcIf + 55PGZAKaZZ3vHyj6JJ1u3q9kfq3o8gcdfnyYy3oCkLAE6YCs+C0X+HLd+KFklekF + 62fyilvn61qB53MvAvlJwgt3AwSrNu9sqjrwaBjEE6xeaDJ/Iv0WqYAWwXcMrHSV + 2/m7n//mUtD1Le2ryhSJV6gjWYxt8aLQbE7gi+cwKFSqFyVtRcK3Gab3kLorISlj + X/SHftB2gXQ6f75W608Fw9c7MX2L2MPMCOzZDTt00AZ4uYshuAdcIaEfmeQHm1ri + alZ7Ccoz2pDYroSjbDv8U2uExf8pnwIDAQABAoICADf7eqgD3GGC1q/Yfzf3qnEq + xXo1+0EkGrEXUmrljHvmM8LYeyvEcerWifkW30SGybzENeHoN3xyhCiTnpUrAz/P + 9/qEUphYOK+SG6xCSTWF427wFb1km2+MEQQRGaFv+A8RRPjVNTYmZAM5wZbYUMz4 + cp+oB3NCL5Xll9lPpo61+pa65mN/1j/vU5TqptM/X5TJrZIke5UbNIF+pP3czNVz + 2RE4oZPbp7YnyDtwqf2jwH55vp8CcY1KemFgPGWAAWnvm7/U5Vjq6ewBSWQl9Y2R + v5bZu9fG61kRViZ6n91EksVVyOLHiNHw4LlGs0LE8a3G+6M2YQzvnHfpXLINhfwU + SZ6BWAJdknVsu6eesYoC08+nyikkq/A3BVD65pT5C9VsmUPbqqpGSYZmAuFgsf9m + zdyKVH4fOPx82DqSZEHZBojg3s5K141DmPp6o0OBX8Ydgfkg2sWXuNi/noBDvh9O + FXWN2IcgK0dET3pX4xFei0QuZgglDp3VyVVSCSUPsOwecZ2XTjtBZPCQVpp3r+QV + LyecFudQ94Ki/0R+M4CrE/mPApDvq+pTjYKFZ10YWtGIdguXq5BVZIMZfZzwIPWN + HdoaFnXRTXTlR4pLIM2nlOvyZmSMo0x6nzUMVGdv4Km9pxi6ZKAgAt4DkbCF9mt0 + QG8RpGJhiIch4kgKFmqxAoIBAQDw4X9Fp9t4f2UiessUDYxLyAtq4acu4ahup5Eb + vlDZPf9gInvz5q9aFHtYgtjTlH449f+EB4isKQscVMysgrJK+7z1IXXMm0sg44wT + F4oV+kvg3KpAridRHyE456RvCPqXYzty6ywJ9B7Zf2oCvd40JUOTm8z11K4COLut + rFIW/24PJA1CWudY/EgzD164k6379On0KryA77iKEZMUztBfHm/bdO8J/zmp7g+E + FS2TCBzR4LpN0uhBwp9wh4rVr74LrPDnQJVZKgeFd24UHEtmcVprAFNUexb2yy1s + vxnHsRPmv5eF7ED1Wlz2K+7LUWqibYOrjeCrS85+CEcey0ApAoIBAQDT2vmbHosb + Qr6ZENt6UX6n0RF8i4g3G4qhucr5hEMQs4H2J8SrUM68QT0GVY0GoDW6f79Pcyr0 + W1tm7qbAOm1Iv4uNYVL1qgpq1GnD5qpWSACGsVSE3OGELlNaVz8fgVdz6zT+rU2A + tp2t795UlrvaLgFI4wITqJF3LoTfy2MZu8JYCzlKM5pZksmEmJfR0RDAot2grtD3 + H5A+PZfUIZ/8BhmdaOAv5i647unfVF6UpPYejZ0rb67oEazxdeIHK3aD5AjurdsO + UpW/PMwsbaltp4iI7hvUfRX7Afb5fPXIhv9pHh1xWYl3djUNWmFoiMMP4tuxpOBo + y+T4maQaiDSHAoIBADrlZ9EIMclMnNXJYE4O4fbFesUvV0lHM3+ayQgXiH0Vg5Nl + 2xjPlqBX0bDajVluPU6AF3GYxfoSLv1GXqTvb9iVpKXrAHp+nef0uxMP9ltZT6Qz + UA1wh3x2OBFJ0hK0B1FsmeSHS8VDQye615jEA8iMM/GrbnnM/p7ccEcOkyO8YJSj + I/rNbzN6u8yAPZCzyx6Hy4w/xsdf1acslOHJj3kyX/cwqCGxnc/GvVR2OSZyHVnT + sLnGj7NEeudwvKlyxuzj5CMmz111wVEI2olgQa9Sl+EBu140mnDNTNYCA7OnwE3z + GoFMOrXC2mf2ZfSge4orbL5Nellnt51pOLp2x8ECggEBALM8Mazw/FOF9mbdgjJM + PFGSaa7rBcVJwdHttDHBmlPI6wzsvFEMPru6nfx76KJQbORqK9r13sN5fyzof59m + TwsbMt/cFSnOQJ39M7YPstDofbl20cDOduUzpEVsRvVKokhqGB3XVRiuZ1y+8WSz + Wh7OiTu3AwzKsrcYXkZQdnlRBq0iYcfLPKzHqUJLLzbOH9Q6djL5c8V/qLNfvNI1 + 2HqKVqV8Ex+zKJhBWRAe+x3bKnbS7MPQ6zNfsOdgCmhydwRCquPzpr7JU/PFZh+4 + b31cHgFrIZR2d2AzW1XcSLzsqa2vUs2RKOIu2deAPaUI/66zCZeTnGBNEFza76Ga + 1oUCggEAA38oXcnputwL103SeD8+uwHjtTf183Rucr+Ryqz6GymiWjlzELqu7TRd + yadAaNg9CuXmYS33Jtk/UNS0k9FvYqGTR+SBXIZr6nt9ZFd0SNlQkwkAQCsuekEs + nJlmUZax7DxXMgIHMKDboHZYM/MhgzEGSALmhU5LZ76MS17v3NEPxYpVHxjAotxW + g03HjWTltS8Bgt6u0KFTGJKEUcfwvWKZtjk5Fc1heZ49zh1nU3zo9C/h8iiijTy2 + s/YksP6cxveae4b7soN4rD/vnfsmKcG+DnTf6B8Zbm6tI2TneYOfFSCryp+yDnaJ + PIDNiTxNecePOmrD+1ivAEXcoL+e1w== + -----END PRIVATE KEY----- + authorizations: + - id: alice + key: fs5wgcer9qj819kfptdlp8gm227ewxnzvsuj9ztycsx08hfhzu + signers: + - webextensions-rsa + - id: bob + key: 9vh6bhlc10y63ow2k4zke7k0c3l9hpr8mo96p92jmbfqngs9e7d + signers: + - webextensions-rsa-with-recommendation + ########################################################################### + # + # The autograph configuration ends here. + # + ########################################################################### + + defaults: &defaults + working_directory: ~/addons-server + docker: + # This is the python version we run in production. + - image: cimg/python:3.8 + # Below are services this project depends on. In addition to these + # services, we also need autograph, which is started in the `test` job + # because we need to pass a configuration file to it and it's not + # possible in this section. + # + # Most settings below should be kept in sync with `docker-compose.yml`. + - image: redis:2.8 + - image: memcached:1.4 + - image: circleci/mysql:8.0 + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: olympia + - image: docker.elastic.co/elasticsearch/elasticsearch:6.8.8 + environment: + # Disable all xpack related features to avoid unrelated logging in + # docker logs. https://github.com/mozilla/addons-server/issues/8887 + xpack.security.enabled: false + xpack.monitoring.enabled: false + xpack.graph.enabled: false + xpack.watcher.enabled: false + discovery.type: single-node + cluster.name: default-cluster + ES_JAVA_OPTS: -Xms256m -Xmx256m + - image: mozilla/autograph:3.3.2 + command: bash -c 'echo -e "$AUTOGRAPH_CONFIG" > amo_config.yaml && cat amo_config.yaml && /go/bin/autograph -c amo_config.yaml' + environment: + AUTOGRAPH_CONFIG: *autograph_config + + defaults-release: &defaults-release + machine: true + working_directory: ~/addons-server + commands: - build_and_push_container_image: + make_release: description: "Builds and pushes a Docker image" parameters: dockerfile: @@ -27,34 +341,163 @@ commands: docker build -t app:build -f << parameters.dockerfile >> --label git.commit="$CIRCLE_SHA1" . docker tag app:build "${DOCKERHUB_REPO}":<< parameters.image_tag >> docker push "${DOCKERHUB_REPO}":<< parameters.image_tag >> + + better_checkout: + description: circle ci checkout step on steroids + parameters: + clone_options: + type: string + default: --depth=1 + description: git clone options + fetch_options: + type: string + default: --depth=10 + description: git fetch options + steps: + - run: + name: checkout + command: | + #!/bin/sh + set -e + + # Workaround old docker images with incorrect $HOME + # check https://github.com/docker/docker/issues/2968 for details + if [ "${HOME}" = "/" ] + then + export HOME=$(getent passwd $(id -un) | cut -d: -f6) + fi + + export SSH_CONFIG_DIR="$HOME/.ssh" + + echo "Using SSH Config Dir $SSH_CONFIG_DIR" + + mkdir -p "$SSH_CONFIG_DIR" + + echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> "$SSH_CONFIG_DIR/known_hosts" + + (umask 077; touch "$SSH_CONFIG_DIR/id_rsa") + chmod 0600 "$SSH_CONFIG_DIR/id_rsa" + (cat $CHECKOUT_KEY > "$SSH_CONFIG_DIR/id_rsa") + + export GIT_SSH_COMMAND='ssh -i $SSH_CONFIG_DIR/id_rsa -o UserKnownHostsFile=$SSH_CONFIG_DIR/known_hosts' + + # use git+ssh instead of https + git config --global url."ssh://git@github.com".insteadOf "https://github.com" || true + git config --global gc.auto 0 || true + + if [ -e .git ] + then + git remote set-url origin "$CIRCLE_REPOSITORY_URL" || true + else + git clone << parameters.clone_options >> "$CIRCLE_REPOSITORY_URL" . + fi + + if [ -n "$CIRCLE_TAG" ] + then + git fetch << parameters.fetch_options >> --force origin "refs/tags/${CIRCLE_TAG}" + else + git fetch << parameters.fetch_options >> --force origin 'circleci:remotes/origin/circleci' + fi + + + if [ -n "$CIRCLE_TAG" ] + then + git reset --hard "$CIRCLE_SHA1" + git checkout -q "$CIRCLE_TAG" + elif [ -n "$CIRCLE_BRANCH" ] + then + git reset --hard "$CIRCLE_SHA1" + git checkout -q -B "$CIRCLE_BRANCH" + fi + + git reset --hard "$CIRCLE_SHA1" + jobs: - build: - machine: true - working_directory: ~/addons-server + test: + <<: *defaults + parameters: + toxenv: + type: string + steps: + - better_checkout + - run: + name: Initial setup + command: | + curl -sL https://dev.mysql.com/get/mysql-apt-config_0.8.15-1_all.deb --output mysql-apt-config.deb + sudo dpkg -i mysql-apt-config.deb + sudo apt-get update -q + sudo apt-get install -y gettext pngcrush librsvg2-bin libmysqlclient-dev + sudo cp ./docker/etc/mime.types /etc/mime.types + sudo touch /addons-server-docker-container + - node/install: + node-version: 12.20.0 + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz + rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz + environment: + DOCKERIZE_VERSION: v0.6.1 + - run: + name: Wait for redis + command: dockerize -wait tcp://localhost:6379 -timeout 1m + - run: + name: Wait for mysql + command: dockerize -wait tcp://localhost:3306 -timeout 1m + - run: + name: Wait for memcached + command: dockerize -wait tcp://localhost:11211 -timeout 1m + - run: + name: Wait for elasticsearch + command: dockerize -wait tcp://localhost:9200 -timeout 1m + - run: pip install --no-deps -r requirements/ci_base.txt + - run: + command: tox -e << parameters.toxenv >> + environment: + ES_VERSION: 6.x + AUTOGRAPH_SERVER_URL: http://127.0.0.1:5500 + + release-master: + <<: *defaults-release steps: - checkout - - build_and_push_container_image: + - make_release: image_tag: latest dockerfile: "Dockerfile.deploy" - build-tag: - machine: true - working_directory: ~/addons-server + release-tag: + <<: *defaults-release steps: - checkout - - build_and_push_container_image: + - make_release: image_tag: "${CIRCLE_TAG}" dockerfile: "Dockerfile.deploy" workflows: version: 2 - build_test_deploy_release: + default-workflow: jobs: - - build: + - test: + matrix: + parameters: + toxenv: + - codestyle + - docs + - assets + - addons-versions-files-ratings + - devhub + - es + - reviewers-and-zadmin + - amo-lib-locales-and-signing + - main + - release-master: filters: branches: only: master - - build-tag: + tags: + ignore: /.*/ + - release-tag: filters: tags: only: /.*/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 51e8cf4be7..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,109 +0,0 @@ -language: python -dist: bionic - -python: - - 3.8 - -addons: - apt: - packages: &global_deps - - cmake - - swig - - elasticsearch - - gettext - - librsvg2-bin - - pngcrush - - uuid - - libgit2-dev - -jobs: - fast_finish: true - include: - - { env: TOXENV=codestyle } - - { env: TOXENV=docs } - - { env: TOXENV=assets } - - { env: TOXENV=addons-versions-files-ratings } - - { env: TOXENV=es ES_VERSION=6.x } - - { env: TOXENV=devhub } - - { env: TOXENV=reviewers-and-zadmin } - - { env: TOXENV=amo-lib-locales-and-signing } - - { env: TOXENV=main } - -env: - global: - - secure: "BGRSmRIIYL+jEKo6nRTwcUZ4m4xiENX4VQqx8blQUsMtpy+XQaQiFwDsPzGyzyAHnbHOAvHBGrWHHRyoJlTqQJziZSZXXp273m6onjYfhmsfGyQoa39flfSlf8mVzSvpf8Te5SdO57scu0dsOt/SAnfRBNOzl1jnOLmO6eqZzHA=" - - AUTOGRAPH_SERVER_URL: http://localhost:5500 - -cache: - pip: true - directories: - - node_modules - - $HOME/.gimme - -services: - - memcached - - redis - -before_install: - - curl -sL https://dev.mysql.com/get/mysql-apt-config_0.8.15-1_all.deb --output mysql-apt-config.deb - - sudo dpkg -i mysql-apt-config.deb - - sudo apt-get update -q - - sudo apt-get install -q -y --allow-unauthenticated -o Dpkg::Options::=--force-confnew mysql-server - - sudo systemctl restart mysql - - sudo mysql_upgrade - - | - if [ $TOXENV == "es" ]; then - if [ $ES_VERSION == "6.x" ]; then - curl -sL https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.8.deb --output elasticsearch.deb - sudo dpkg -i --force-confnew elasticsearch.deb - sudo sed -i.old 's/-Xms1g/-Xms128m/' /etc/elasticsearch/jvm.options - sudo sed -i.old 's/-Xmx1g/-Xmx128m/' /etc/elasticsearch/jvm.options - echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlog4j.skipJansi=true\n-server\n' | sudo tee -a /etc/elasticsearch/jvm.options - sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch - fi - sudo systemctl start elasticsearch - fi - - mysql -e 'create database olympia;' - - export GOPATH=$HOME/go - - export PATH=$HOME/usr/local/go/bin:$GOPATH/bin:$PATH - - sudo cp ./docker/etc/mime.types /etc/mime.types - -install: - - nvm current - - nvm deactivate - - nvm install 10 - - nvm use 10 - - pip install --no-deps -r requirements/travis_base.txt - -before_script: - - mysql --version - - node --version - - java -version - - | - if [ $TOXENV == "es" ]; then - curl --retry 3 --retry-delay 10 --retry-connrefused http://localhost:9200/; - fi - - sudo touch /addons-server-docker-container - -script: - - | - if [ $TRAVIS_EVENT_TYPE != "cron" ]; then - if [ $TOXENV == "amo-lib-locales-and-signing" ] || [ $TOXENV == "reviewers-and-zadmin" ] ; then - docker run --name autograph -d -p 5500:5500 -v $(pwd)/scripts/:/scripts/ mozilla/autograph:3.3.2 /go/bin/autograph -c /scripts/autograph_travis_test_config.yaml - fi - RUNNING_IN_CI=True tox - fi - -after_script: - - docker stop autograph - -notifications: - slack: - if: branch = master - rooms: - - secure: VuUiui/fUMV6cXrYpTXrzLnYurcsJQOeczWA2rvsq8fCFjSe4MXMgv/kF/2b7F7O8mmEgQAUGozJAaChmYCiwDFOtki7bUBGl6yOm5OWW1ZnkTxObXB7pKG/aFT0VoF4EKIRp46M4tT8yqZ0m+CXP8/rITE0aLJoz/xmGzOaCc8= - on_success: change - on_failure: always - -git: - depth: 1 diff --git a/README.rst b/README.rst index 29de5d60e2..e2e0a56c73 100644 --- a/README.rst +++ b/README.rst @@ -2,8 +2,8 @@ :target: https://github.com/mozilla/addons-server/blob/master/.github/CODE_OF_CONDUCT.md :alt: Code of conduct -.. image:: https://travis-ci.org/mozilla/addons-server.svg?branch=master - :target: https://travis-ci.org/mozilla/addons-server +.. image:: https://circleci.com/gh/mozilla/addons-server.svg?style=svg + :target: https://circleci.com/gh/mozilla/addons-server Addons-Server diff --git a/contribute.json b/contribute.json index bce0ba6edd..12bd0c06c8 100644 --- a/contribute.json +++ b/contribute.json @@ -4,12 +4,12 @@ "repository": { "url": "https://github.com/mozilla/addons-server", "license": "BSD 3-Clause", - "tests": "https://travis-ci.org/mozilla/addons-server" + "tests": "https://app.circleci.com/pipelines/github/mozilla/addons-server" }, "participate": { "home": "https://wiki.mozilla.org/Add-ons/Contribute/AMO/Code", "docs": "http://addons-server.readthedocs.io/", - "matrix": "https://chat.mozilla.org/#/room/#amo:mozilla.org", + "matrix": "https://chat.mozilla.org/#/room/#amo:mozilla.org" }, "bugs": { "list": "https://github.com/mozilla/addons-server/issues", diff --git a/docker-compose.yml b/docker-compose.yml index f69cae4604..6741801cb1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,7 +89,7 @@ services: autograph: image: mozilla/autograph:3.3.2 - command: /go/bin/autograph -c /code/scripts/autograph_travis_test_config.yaml + command: /go/bin/autograph -c /code/scripts/autograph_localdev_config.yaml volumes: - .:/code diff --git a/requirements/travis_base.txt b/requirements/ci_base.txt similarity index 100% rename from requirements/travis_base.txt rename to requirements/ci_base.txt diff --git a/scripts/autograph_travis_test_config.yaml b/scripts/autograph_localdev_config.yaml similarity index 98% rename from scripts/autograph_travis_test_config.yaml rename to scripts/autograph_localdev_config.yaml index 5705056adb..4e53ac6f09 100644 --- a/scripts/autograph_travis_test_config.yaml +++ b/scripts/autograph_localdev_config.yaml @@ -1,5 +1,8 @@ -# Note: Most of the configuration here got copied from +# Note (1): Most of the configuration here got copied from # https://github.com/mozilla-services/autograph/blob/master/autograph.yaml +# +# Note (2): the content of the file is also embedded in `.circleci/config.yml`. +# Any change here should likely be duplicated. server: # This port should be perfectly free, the upstream default of 8000 diff --git a/src/olympia/amo/search.py b/src/olympia/amo/search.py index cea41d5291..fb380f37e0 100644 --- a/src/olympia/amo/search.py +++ b/src/olympia/amo/search.py @@ -1,5 +1,3 @@ -import os - from django.conf import settings as dj_settings from django_statsd.clients import statsd @@ -26,9 +24,6 @@ def get_es(hosts=None, timeout=None, **settings): else getattr(dj_settings, 'ES_TIMEOUT', DEFAULT_TIMEOUT) ) - if os.environ.get('RUNNING_IN_CI'): - settings['http_auth'] = ('elastic', 'changeme') - return Elasticsearch(hosts, timeout=timeout, **settings) diff --git a/src/olympia/lib/settings_base.py b/src/olympia/lib/settings_base.py index 9066a77871..c321d5cd84 100644 --- a/src/olympia/lib/settings_base.py +++ b/src/olympia/lib/settings_base.py @@ -114,7 +114,7 @@ CORS_URLS_REGEX = r'{}(?!accounts/session/)'.format(DRF_API_REGEX) def get_db_config(environ_var, atomic_requests=True): - values = env.db(var=environ_var, default='mysql://root:@localhost/olympia') + values = env.db(var=environ_var, default='mysql://root:@127.0.0.1/olympia') values.update( { diff --git a/src/olympia/reviewers/tests/test_serializers.py b/src/olympia/reviewers/tests/test_serializers.py index 6ea1ec3608..7d3073e8ba 100644 --- a/src/olympia/reviewers/tests/test_serializers.py +++ b/src/olympia/reviewers/tests/test_serializers.py @@ -710,7 +710,7 @@ class TestAddonCompareVersionSerializer(TestCase): assert readme_data['status'] == 'D' assert readme_data['depth'] == 0 assert readme_data['filename'] == 'README.md' - # Not testing mimetype as text/markdown is missing in travis mimetypes + # Not testing mimetype as text/markdown is missing in CI mimetypes # database. But it doesn't matter much here since we're primarily # after the git status. assert readme_data['mime_category'] is None diff --git a/src/olympia/versions/tests/test_utils.py b/src/olympia/versions/tests/test_utils.py index 7bea5e32d8..fe1fabec47 100644 --- a/src/olympia/versions/tests/test_utils.py +++ b/src/olympia/versions/tests/test_utils.py @@ -38,7 +38,7 @@ def test_write_svg_to_png(filename): out = os.path.join(out_dir, 'a', 'b.png') write_svg_to_png(svg, out) assert storage.exists(out) - # compare the image content. rms should be 0 but travis renders it + # compare the image content. rms should be 0 but CI renders it # different... 3 is the magic difference. svg_png_img = Image.open(svg_png) svg_out_img = Image.open(out)