Initial implementation of github-nginx-cache (#1)
This commit is contained in:
billytrend
GitHub
Родитель
83779dcac3
Коммит
67cc01d2b2
|
|
@ -0,0 +1,26 @@
|
|||
pool:
|
||||
vmImage: "Ubuntu 16.04"
|
||||
|
||||
trigger:
|
||||
- master
|
||||
|
||||
steps:
|
||||
- task: Docker@2
|
||||
displayName: Build docker container
|
||||
inputs:
|
||||
command: build
|
||||
arguments: -t testbuild
|
||||
|
||||
- task: Docker@2
|
||||
displayName: Run docker container
|
||||
inputs:
|
||||
command: run
|
||||
arguments: -p 8000:80 -d testbuild
|
||||
|
||||
- script: npm -s ci
|
||||
displayName: "Install dependencies"
|
||||
workingDirectory: ./test
|
||||
|
||||
- script: npm run -s test
|
||||
displayName: "Run tests"
|
||||
workingDirectory: ./test
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
trigger:
|
||||
- master
|
||||
|
||||
pr: none
|
||||
|
||||
name: v1.$(Date:yyyy-MM-dd)$(Rev:.r)
|
||||
stages:
|
||||
- stage: Build_Docker
|
||||
jobs:
|
||||
- job: Build_Docker
|
||||
displayName: Build and publish to hub.docker.com
|
||||
pool:
|
||||
vmImage: "Ubuntu-16.04"
|
||||
steps:
|
||||
- task: Docker@2
|
||||
displayName: "Build azuredevx/github-nginx-cache"
|
||||
inputs:
|
||||
containerRegistry: "github-nginx-cache docker"
|
||||
repository: "azuredevx/github-nginx-cache"
|
||||
tags: |
|
||||
$(Build.BuildNumber)
|
||||
latest
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
*
|
||||
!nginx-config
|
||||
!kubernetes/active-passive
|
||||
|
|
@ -1,330 +1,3 @@
|
|||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
||||
|
||||
# User-specific files
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
|
||||
# Visual Studio 2015/2017 cache/options directory
|
||||
.vs/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# Visual Studio 2017 auto generated files
|
||||
Generated\ Files/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUNIT
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# Benchmark Results
|
||||
BenchmarkDotNet.Artifacts/
|
||||
|
||||
# .NET Core
|
||||
project.lock.json
|
||||
project.fragment.lock.json
|
||||
artifacts/
|
||||
**/Properties/launchSettings.json
|
||||
|
||||
# StyleCop
|
||||
StyleCopReport.xml
|
||||
|
||||
# Files built by Visual Studio
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_i.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.iobj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.ipdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*.log
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
*.VC.db
|
||||
*.VC.VC.opendb
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# Visual Studio Trace Files
|
||||
*.e2e
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# JustCode is a .NET coding add-in
|
||||
.JustCode
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# AxoCover is a Code Coverage Tool
|
||||
.axoCover/*
|
||||
!.axoCover/settings.json
|
||||
|
||||
# Visual Studio code coverage results
|
||||
*.coverage
|
||||
*.coveragexml
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
# Note: Comment the next line if you want to checkin your web deploy settings,
|
||||
# but database connection strings (with potential passwords) will be unencrypted
|
||||
*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||
# in these scripts will be unencrypted
|
||||
PublishScripts/
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
# The packages folder can be ignored because of Package Restore
|
||||
**/[Pp]ackages/*
|
||||
# except build/, which is used as an MSBuild target.
|
||||
!**/[Pp]ackages/build/
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/[Pp]ackages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignorable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Windows Store app package directories and files
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
Package.StoreAssociation.xml
|
||||
_pkginfo.txt
|
||||
*.appx
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.jfm
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
orleans.codegen.cs
|
||||
|
||||
# Including strong name files can present a security risk
|
||||
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
||||
#*.snk
|
||||
|
||||
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||
#bower_components/
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
ServiceFabricBackup/
|
||||
*.rptproj.bak
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
*.ndf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
*.rptproj.rsuser
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
node_modules/
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||
*.vbw
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
paket-files/
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
# JetBrains Rider
|
||||
.idea/
|
||||
*.sln.iml
|
||||
|
||||
# CodeRush
|
||||
.cr/
|
||||
|
||||
# Python Tools for Visual Studio (PTVS)
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Cake - Uncomment if you are using it
|
||||
# tools/**
|
||||
# !tools/packages.config
|
||||
|
||||
# Tabs Studio
|
||||
*.tss
|
||||
|
||||
# Telerik's JustMock configuration file
|
||||
*.jmconfig
|
||||
|
||||
# BizTalk build output
|
||||
*.btp.cs
|
||||
*.btm.cs
|
||||
*.odx.cs
|
||||
*.xsd.cs
|
||||
|
||||
# OpenCover UI analysis results
|
||||
OpenCover/
|
||||
|
||||
# Azure Stream Analytics local run output
|
||||
ASALocalRun/
|
||||
|
||||
# MSBuild Binary and Structured Log
|
||||
*.binlog
|
||||
|
||||
# NVidia Nsight GPU debugger configuration file
|
||||
*.nvuser
|
||||
|
||||
# MFractors (Xamarin productivity tool) working folder
|
||||
.mfractor/
|
||||
nginx-logs
|
||||
**/node_modules
|
||||
**/buildcache
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"folders": [
|
||||
{
|
||||
"path": "/Users/billytrend/Repositories/nginx-cache"
|
||||
},
|
||||
{
|
||||
"path": "/Users/billytrend/Repositories/nginx-cache/test"
|
||||
}
|
||||
],
|
||||
"settings": {}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
FROM nginx
|
||||
|
||||
RUN apt-get update; apt-get install -y curl
|
||||
|
||||
COPY ./nginx-config/ /etc/nginx/
|
||||
COPY kubernetes/active-passive/check-readiness.sh /check-readiness.sh
|
||||
|
||||
# test nginx config
|
||||
RUN nginx -c /etc/nginx/nginx.conf -t
|
||||
70
README.md
70
README.md
|
|
@ -1,7 +1,75 @@
|
|||
## Github nginx cache
|
||||
|
||||
This repo contains nginx configuration tuned to sit in front of github endpoints and provide caching functionality. Github will not rate-limit [conditional requests](https://developer.github.com/v3/#conditional-requests). The [`proxy_cache_*` nginx directives](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache) force nginx to revalidate any cached content from the upstream server (in this case, github). Revalidation is performed by nginx as a conditional request, therefore it will not reduce api limits. This works for both authenticated and unauthenticated requests.
|
||||
|
||||
Here is an example how rate-limiting is mitigated for unauthenticated requests against both https://api.github.com and the cache running on http://localhost:8000.
|
||||
|
||||
|
||||
|
||||
### Quick Start
|
||||
|
||||
docker run -d -p 8000:80 azure-devex/github-nginx-cache
|
||||
curl localhost:8000/api/repos/azure/github-nginx-cache
|
||||
|
||||
The github domains are mapped as follows:
|
||||
|
||||
| Github URL | Cache URL |
|
||||
| ---------------------------- | -------------------------- |
|
||||
| api.github.com/\* | localhost:8000/api/\* |
|
||||
| raw.githubusercontent.com/\* | localhost:8000/raw/\* |
|
||||
| codeload.github.com/\* | localhost:8000/codeload/\* |
|
||||
|
||||
### CI/CD
|
||||
|
||||
Docker publish [](https://dev.azure.com/azure-sdk/public/_build/latest?definitionId=496&branchName=master)
|
||||
|
||||
## Develop
|
||||
|
||||
### Build
|
||||
|
||||
docker build .
|
||||
|
||||
### Debug
|
||||
|
||||
#### Fish
|
||||
|
||||
docker build -t custom-nginx . && docker run -it -p 8000:80 -v (pwd)/nginx-logs:/var/log/nginx custom-nginx
|
||||
curl localhost:8000/health/alive
|
||||
|
||||
#### Bash
|
||||
|
||||
docker build -t custom-nginx . && docker run -it -p 8000:80 -v $(pwd)/nginx-logs:/var/log/nginx custom-nginx
|
||||
curl localhost:8000/health/alive
|
||||
|
||||
### Test
|
||||
|
||||
# Run image on localhost:8000
|
||||
cd test
|
||||
npm ci
|
||||
npm run test
|
||||
|
||||
## Implementation details
|
||||
|
||||
### Github consistency
|
||||
|
||||
The cache is designed for the highest possible github consistency such that it ignores any `Cache-Control` headers that github sends and forces nginx to REVALIDATE for every request. A limitation in nginx means that the lowest value for `proxy_cache_valid` directive is one second. This means that two identical requests to github within the space of one second will HIT (return cached response without revalidating) rather than REVALIDATE.
|
||||
|
||||
### Cache partitioning
|
||||
|
||||
The cache may be used for complicated applications where multiple app and oauth tokens are being used to access github. The default behaviour in this case is to parition the cache by token. This means that a request with token A will not leverage any cached content from requests using token B.
|
||||
|
||||
This behaviour is for the following reasons.
|
||||
|
||||
1. Security - there are edge cases in which using two tokens within one second of each other could cause a response to be leaked to the second request even if the second token was not allowed to access the resource.
|
||||
1. Prevent cache churn - if the cache was not partitioned, multiple requests to one api route with different tokens may cause the cache to be evacuated unnecessarily if these tokens have different access permissions.
|
||||
|
||||
There may be cases however where this behaviour needs to be overridden at the discretion of the client. For example when using a GitHub app, the token may expire every hour or so in which case the default behaviour would be for the cache to reset every hour which is not desirable.
|
||||
|
||||
By setting **X-Cache-Key** header, the cache [will be paritioned](nginx-config/cache_key_logic.conf) on this arbitrary string rather than the token.
|
||||
|
||||
# Contributing
|
||||
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
||||
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
|
||||
the rights to use your contribution. For details, visit https://cla.microsoft.com.
|
||||
|
||||
|
|
|
|||
Двоичный файл не отображается.
|
После Ширина: | Высота: | Размер: 72 KiB |
|
|
@ -0,0 +1,28 @@
|
|||
#!/bin/bash
|
||||
|
||||
podhealth() {
|
||||
if [[ -z "${podname}" ]]; then
|
||||
echo "podname not set"
|
||||
return 1
|
||||
else
|
||||
echo "podname is set to ${podname}"
|
||||
fi
|
||||
|
||||
activepod=$(for i in 0 1; do echo $podname-$i;done | grep -v $HOSTNAME)
|
||||
|
||||
echo "activepod = ${activepod}.${serviceDomain}"
|
||||
|
||||
# ${activepod}.${serviceDomain} is the FQDN of a pod
|
||||
# eg mypod.myservice.mynamespace.svc.cluster.local
|
||||
curl -I ${activepod}.${serviceDomain}
|
||||
|
||||
# if curl fails then we need to be active otherwise stay passive
|
||||
if [ $? -eq 0 ]
|
||||
then
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
podhealth
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: github-nginx-cache-loadbalancer
|
||||
spec:
|
||||
ports:
|
||||
- port: 80
|
||||
protocol: TCP
|
||||
targetPort: 80
|
||||
selector:
|
||||
app: github-nginx-cache
|
||||
type: LoadBalancer
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: github-nginx-cache
|
||||
labels:
|
||||
app: github-nginx-cache
|
||||
spec:
|
||||
containers:
|
||||
- image: azuredevx/github-nginx-cache:latest # specify the release for production use
|
||||
name: mypod
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
volumeMounts:
|
||||
- name: volume
|
||||
mountPath: /mnt/github-nginx-cache
|
||||
volumes:
|
||||
- name: volume
|
||||
persistentVolumeClaim:
|
||||
claimName: managed-disk
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: managed-disk
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: managed-premium
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
set $cache_key $http_authorization;
|
||||
|
||||
# if x-cache-key header is present, use this to partition the nginx cache
|
||||
# instead of the users token
|
||||
if ($http_x_cache_key) {
|
||||
set $cache_key $http_x_cache_key;
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
expires max;
|
||||
# ignore the cache control headers that git sends so that we can revalidate more aggresively
|
||||
proxy_ignore_headers Cache-Control;
|
||||
# enable revalidation
|
||||
proxy_cache_revalidate on;
|
||||
# enable request coalescing so multiple requests are combined into one
|
||||
proxy_cache_lock on;
|
||||
proxy_cache_lock_timeout 2s;
|
||||
# cache is valid for max 1s so we can be out of sync with github for max that amount of time
|
||||
# nginx does not support a lower value for this
|
||||
proxy_cache_valid 1s;
|
||||
# cache storage config
|
||||
proxy_cache cache_zone;
|
||||
# use stale response if github is not available
|
||||
proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
|
||||
# add x-cached header to responses to report how the cache was used
|
||||
add_header X-Cached $upstream_cache_status;
|
||||
# cache key
|
||||
proxy_cache_key $cache_key:$request_to_proxy:$is_args:$args;
|
||||
# rewrite the host header so github is not confused
|
||||
proxy_set_header HOST $downstream_url;
|
||||
# set target for proxying
|
||||
proxy_pass https://$downstream_url$request_to_proxy$is_args$args;
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
location ~ /api(?<request_to_proxy>\/.*) {
|
||||
set $downstream_url "api.github.com";
|
||||
|
||||
include cache_key_logic.conf;
|
||||
include common_cache_settings.conf;
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
location ~ /codeload(?<request_to_proxy>\/.*) {
|
||||
set $downstream_url "codeload.github.com";
|
||||
|
||||
include cache_key_logic.conf;
|
||||
include common_cache_settings.conf;
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
location ~ /raw(?<request_to_proxy>\/.*) {
|
||||
set $downstream_url "raw.githubusercontent.com";
|
||||
|
||||
include cache_key_logic.conf;
|
||||
include common_cache_settings.conf;
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
location /health/alive {
|
||||
return 200 '{ "message": "Github cache is alive" }';
|
||||
add_header Content-Type application/json;
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
user nginx;
|
||||
|
||||
# Set number of worker processes automatically based on number of CPU cores.
|
||||
worker_processes auto;
|
||||
|
||||
# Enables the use of JIT for regular expressions to speed-up their processing.
|
||||
pcre_jit on;
|
||||
|
||||
# Configures default error logger.
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
|
||||
# Includes files with directives to load dynamic modules.
|
||||
include /etc/nginx/modules/*.conf;
|
||||
|
||||
events {
|
||||
# The maximum number of simultaneous connections that can be opened by
|
||||
# a worker process.
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
# Includes mapping of file name extensions to MIME types of responses
|
||||
# and defines the default type.
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
# Don't tell nginx version to clients.
|
||||
server_tokens off;
|
||||
|
||||
# Specifies the maximum accepted body size of a client request, as
|
||||
# indicated by the request header Content-Length. If the stated content
|
||||
# length is greater than this size, then the client receives the HTTP
|
||||
# error code 413. Set to 0 to disable.
|
||||
client_max_body_size 1m;
|
||||
|
||||
# Timeout for keep-alive connections. Server will close connections after
|
||||
# this time.
|
||||
keepalive_timeout 65;
|
||||
|
||||
# Sendfile copies data between one FD and other from within the kernel,
|
||||
# which is more efficient than read() + write().
|
||||
sendfile on;
|
||||
|
||||
# Don't buffer data-sends (disable Nagle algorithm).
|
||||
# Good for sending frequent small bursts of data in real time.
|
||||
tcp_nodelay on;
|
||||
|
||||
# Specifies that our cipher suits should be preferred over client ciphers.
|
||||
ssl_prefer_server_ciphers on;
|
||||
|
||||
# Enables a shared SSL cache with size that can hold around 8000 sessions.
|
||||
ssl_session_cache shared:SSL:2m;
|
||||
|
||||
# Set the Vary HTTP header as defined in the RFC 2616.
|
||||
gzip_vary on;
|
||||
|
||||
|
||||
# Specifies the main log format
|
||||
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
# Specifies the main log location
|
||||
access_log /var/log/nginx/access.log access;
|
||||
|
||||
# logs proxy pass information
|
||||
log_format upstreamlog '[$time_local] $remote_addr - $remote_user - $server_name to: $upstream_addr: $request upstream_response_time $upstream_response_time msec $msec request_time $request_time';
|
||||
# Specifies the upstream log location
|
||||
access_log /var/log/nginx/upstream.log upstreamlog;
|
||||
|
||||
# Specifies the cache_status log format.
|
||||
log_format cache_status '[$time_local] "$request" $upstream_cache_status';
|
||||
# Specifies the cache_status log location
|
||||
access_log /var/log/nginx/cache_access.log cache_status;
|
||||
|
||||
# location and settings
|
||||
proxy_cache_path /mnt/github-nginx-cache use_temp_path=off levels=1:2 keys_zone=cache_zone:200m max_size=200g inactive=1y;
|
||||
|
||||
log_format json_combined escape=json
|
||||
'{'
|
||||
'"time_local":"$time_local",'
|
||||
'"remote_addr":"$remote_addr",'
|
||||
'"request_method":"$request_method",'
|
||||
'"request_uri":"$request_uri",'
|
||||
'"status": "$status",'
|
||||
'"body_bytes_sent":"$body_bytes_sent",'
|
||||
'"request_time":"$request_time",'
|
||||
'"http_referrer":"$http_referer",'
|
||||
'"upstream_addr":"$upstream_addr",'
|
||||
'"upstream_response_time":"$upstream_response_time",'
|
||||
'"msec":"$msec",'
|
||||
'"upstream_http_x_ratelimit_limit":"$upstream_http_x_ratelimit_limit",'
|
||||
'"upstream_http_x_ratelimit_remaining":"$upstream_http_x_ratelimit_remaining",'
|
||||
'"upstream_http_x_ratelimit_reset":"$upstream_http_x_ratelimit_reset",'
|
||||
'"upstream_http_retry_after":"$upstream_http_retry_after",'
|
||||
'"request_to_proxy":"$request_to_proxy$is_args$args",'
|
||||
'"http_x_cache_key":"$http_x_cache_key",'
|
||||
'"downstream_url":"$downstream_url",'
|
||||
'"upstream_status":"$upstream_status",'
|
||||
'"upstream_bytes_received":"$upstream_bytes_received",'
|
||||
'"upstream_bytes_sent":"$upstream_bytes_sent",'
|
||||
'"upstream_cache_status":"$upstream_cache_status",'
|
||||
'"request_body":"$request_body",'
|
||||
'"http_user_agent":"$http_user_agent"'
|
||||
'}';
|
||||
|
||||
access_log /dev/stdout json_combined;
|
||||
error_log /dev/stderr warn;
|
||||
|
||||
server {
|
||||
resolver 8.8.8.8;
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
location / {
|
||||
return 200 'This is a caching service for gitub: https://github.com/Azure/github-nginx-cache/';
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
|
||||
include health_check.conf;
|
||||
include github-domain-configs/*.conf;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
trailingComma: "all"
|
||||
printWidth: 120
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
// @ts-check
|
||||
|
||||
/** @type {jest.InitialOptions} */
|
||||
const config = {
|
||||
transform: {
|
||||
"^.+\\.ts$": "ts-jest",
|
||||
},
|
||||
moduleFileExtensions: ["ts", "js", "json", "node"],
|
||||
moduleNameMapper: {},
|
||||
collectCoverage: false,
|
||||
globals: {
|
||||
"ts-jest": {
|
||||
tsConfig: "tsconfig.json",
|
||||
},
|
||||
},
|
||||
setupFiles: ["<rootDir>/jest-setup.ts"],
|
||||
testMatch: ["<rootDir>/src/**/*.test.ts"],
|
||||
verbose: true,
|
||||
testEnvironment: "node",
|
||||
};
|
||||
|
||||
module.exports = config;
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -0,0 +1,45 @@
|
|||
{
|
||||
"name": "github-nginx-cache-test",
|
||||
"version": "0.1.0",
|
||||
"description": "This project welcomes contributions and suggestions. Most contributions require you to agree to a\r Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us\r the rights to use your contribution. For details, visit https://cla.microsoft.com.",
|
||||
"scripts": {
|
||||
"test": "jest",
|
||||
"test:ci": "jest --ci --reporters=default --reporters=jest-junit",
|
||||
"lint": "tslint -p tsconfig.json",
|
||||
"test:watch": "jest --watch --coverage=false --config ./config/jest.dev.config.js"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/Azure/github-nginx-cache.git"
|
||||
},
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"bugs": {
|
||||
"url": "https://github.com/Azure/github-nginx-cache/issues"
|
||||
},
|
||||
"homepage": "https://github.com/Azure/github-nginx-cache#readme",
|
||||
"devDependencies": {
|
||||
"@types/convict": "^4.2.1",
|
||||
"@types/jest": "^24.0.13",
|
||||
"@types/node-fetch": "^2.3.4",
|
||||
"jest": "^24.8.0",
|
||||
"jest-junit": "^6.4.0",
|
||||
"prettier": "^1.17.1",
|
||||
"ts-jest": "^24.0.2",
|
||||
"tslint": "^5.16.0",
|
||||
"tslint-config-prettier": "^1.18.0",
|
||||
"tslint-plugin-prettier": "^2.0.1",
|
||||
"typescript": "^3.4.5"
|
||||
},
|
||||
"dependencies": {
|
||||
"@octokit/rest": "^16.27.0",
|
||||
"@types/node": "^12.0.2",
|
||||
"convict": "^5.0.0",
|
||||
"node-fetch": "^2.6.0",
|
||||
"nodegit": "^0.24.3"
|
||||
},
|
||||
"jest-junit": {
|
||||
"outputDirectory": ".",
|
||||
"outputName": "test-results.xml"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
import Github from "@octokit/rest";
|
||||
|
||||
import { configuration } from "./config";
|
||||
import { xCacheKey } from "./x-cache-key-plugin";
|
||||
import { delay, getHeader } from "./utils";
|
||||
|
||||
Github.plugin(xCacheKey);
|
||||
|
||||
describe("API", () => {
|
||||
const github = new Github({
|
||||
baseUrl: configuration.localApiProxyUrl,
|
||||
});
|
||||
|
||||
it("should go to github", async () => {
|
||||
const a = await github.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": Math.random() },
|
||||
} as any);
|
||||
expect(getHeader(a, "x-cached")).toMatchInlineSnapshot(`"MISS"`);
|
||||
});
|
||||
|
||||
it("should HIT for immediate re-request", async () => {
|
||||
const key = Math.random();
|
||||
|
||||
const a = await github.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": key },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(a, "x-cached")).toMatchInlineSnapshot(`"MISS"`);
|
||||
|
||||
const b = await github.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": key },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(b, "x-cached")).toMatchInlineSnapshot(`"HIT"`);
|
||||
});
|
||||
|
||||
it("should REVALIDATE after ~2 seconds", async () => {
|
||||
const key = Math.random();
|
||||
|
||||
const a = await github.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": key },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(a, "x-cached")).toMatchInlineSnapshot(`"MISS"`);
|
||||
|
||||
await delay(2000);
|
||||
|
||||
const b = await github.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": key },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(b, "x-cached")).toMatchInlineSnapshot(`"REVALIDATED"`);
|
||||
});
|
||||
|
||||
it("REVALIDATE should not affect API limit", async () => {
|
||||
const key = Math.random();
|
||||
|
||||
const a = await github.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": key },
|
||||
} as any);
|
||||
|
||||
await delay(2000);
|
||||
|
||||
const b = await github.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": key },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(a, "x-ratelimit-remaining")).toEqual(getHeader(b, "x-ratelimit-remaining"));
|
||||
});
|
||||
|
||||
it("requests with different x-cache-key should have different caches", async () => {
|
||||
const a = await github.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": Math.random() },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(a, "x-cached")).toMatchInlineSnapshot(`"MISS"`);
|
||||
|
||||
const b = await github.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": Math.random() },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(b, "x-cached")).toMatchInlineSnapshot(`"MISS"`);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
import { configSchema } from "./schema";
|
||||
|
||||
configSchema.validate({ allowed: "strict" });
|
||||
|
||||
export const configuration = configSchema.getProperties();
|
||||
|
|
@ -0,0 +1 @@
|
|||
export * from "./configuration";
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
import convict from "convict";
|
||||
|
||||
export type Env = "test";
|
||||
|
||||
export const configSchema = convict({
|
||||
env: {
|
||||
doc: "The application environment.",
|
||||
format: ["test"],
|
||||
default: "test",
|
||||
env: "NODE_ENV",
|
||||
},
|
||||
localApiProxyUrl: {
|
||||
doc: "URL for caching api proxy",
|
||||
format: "url",
|
||||
default: "http://localhost:8000/api/", // *.vcap.me resolves to 127.0.0.1 so it can be used for testing subdomain requests to localhost
|
||||
env: "localApiProxyUrl",
|
||||
},
|
||||
});
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
import Github from "@octokit/rest";
|
||||
|
||||
export interface NginxHeaders {
|
||||
"x-cached": "REVALIDATED" | "MISS" | "HIT";
|
||||
}
|
||||
|
||||
export const getHeader = (
|
||||
response: Github.Response<unknown>,
|
||||
header: keyof (NginxHeaders & Github.Response<any>["headers"]),
|
||||
) => {
|
||||
return ((response.headers as unknown) as NginxHeaders & Github.Response<any>["headers"])[header];
|
||||
};
|
||||
|
||||
export const delay = (ms: number) => new Promise(res => setTimeout(res, ms));
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
import Github from "@octokit/rest";
|
||||
|
||||
import { configuration } from "./config";
|
||||
import { getHeader } from "./utils";
|
||||
import { xCacheKey } from "./x-cache-key-plugin";
|
||||
|
||||
describe("xCacheKey", () => {
|
||||
it("requests with different x-cache-key should have different caches", async () => {
|
||||
const githubA = new (Github.plugin([xCacheKey]))({
|
||||
baseUrl: configuration.localApiProxyUrl,
|
||||
cacheKey: "a",
|
||||
});
|
||||
|
||||
const a = await githubA.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
request: { cacheKey: Math.random() },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(a, "x-cached")).toMatchInlineSnapshot(`"MISS"`);
|
||||
|
||||
const githubB = new Github({
|
||||
baseUrl: configuration.localApiProxyUrl,
|
||||
cacheKey: "a",
|
||||
});
|
||||
|
||||
const b = await githubB.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": Math.random() },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(b, "x-cached")).toMatchInlineSnapshot(`"MISS"`);
|
||||
});
|
||||
|
||||
it("requests with same x-cache-key should have different caches", async () => {
|
||||
const githubA = new Github({
|
||||
baseUrl: configuration.localApiProxyUrl,
|
||||
cacheKey: "a",
|
||||
});
|
||||
|
||||
const a = await githubA.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": Math.random() },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(a, "x-cached")).toMatchInlineSnapshot(`"MISS"`);
|
||||
|
||||
const githubB = new Github({
|
||||
baseUrl: configuration.localApiProxyUrl,
|
||||
cacheKey: "a",
|
||||
});
|
||||
|
||||
const b = await githubB.repos.get({
|
||||
owner: "octocat",
|
||||
repo: "Hello-World",
|
||||
headers: { "x-cache-key": Math.random() },
|
||||
} as any);
|
||||
|
||||
expect(getHeader(b, "x-cached")).toMatchInlineSnapshot(`"MISS"`);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
import Octokit from "@octokit/rest";
|
||||
|
||||
export const xCacheKey: Octokit.Plugin = (octokit, options: { cacheKey?: string } & Octokit.Options) => {
|
||||
octokit.hook.wrap("request", async (request, requestOptions) => {
|
||||
const cacheKey: string | undefined = options.cacheKey || (requestOptions.request as any).cacheKey;
|
||||
|
||||
const augmentedOptions = cacheKey
|
||||
? {
|
||||
...requestOptions,
|
||||
headers: { ...requestOptions.headers, "x-cache-key": cacheKey },
|
||||
}
|
||||
: requestOptions;
|
||||
|
||||
const response = await request(augmentedOptions);
|
||||
|
||||
return response;
|
||||
});
|
||||
};
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
/* Basic Options */
|
||||
"target": "es2019" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */,
|
||||
"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */,
|
||||
// "lib": [], /* Specify library files to be included in the compilation. */
|
||||
// "allowJs": true, /* Allow javascript files to be compiled. */
|
||||
// "checkJs": true, /* Report errors in .js files. */
|
||||
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
|
||||
// "declaration": true, /* Generates corresponding '.d.ts' file. */
|
||||
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
|
||||
// "sourceMap": true, /* Generates corresponding '.map' file. */
|
||||
// "outFile": "./", /* Concatenate and emit output to single file. */
|
||||
"rootDir": "src",
|
||||
"outDir": "bin",
|
||||
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
|
||||
// "composite": true, /* Enable project compilation */
|
||||
// "incremental": true, /* Enable incremental compilation */
|
||||
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
|
||||
// "removeComments": true, /* Do not emit comments to output. */
|
||||
// "noEmit": true, /* Do not emit outputs. */
|
||||
// "importHelpers": true, /* Import emit helpers from 'tslib'. */
|
||||
// "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
|
||||
// "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
|
||||
|
||||
/* Strict Type-Checking Options */
|
||||
"strict": true /* Enable all strict type-checking options. */,
|
||||
|
||||
"incremental": true,
|
||||
"tsBuildInfoFile": "./buildcache/backend.tsbuildinfo",
|
||||
|
||||
/* Additional Checks */
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noImplicitReturns": true,
|
||||
|
||||
"allowSyntheticDefaultImports": true /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */,
|
||||
"esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
|
||||
|
||||
/* Source Map Options */
|
||||
"sourceMap": true,
|
||||
|
||||
/* Experimental Options */
|
||||
"experimentalDecorators": true /* Enables experimental support for ES7 decorators. */,
|
||||
"emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */
|
||||
},
|
||||
"include": ["src/**/*.ts", "src/definitions.d.ts"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
{
|
||||
"defaultSeverity": "error",
|
||||
"extends": ["tslint:latest", "tslint-config-prettier", "tslint-plugin-prettier"],
|
||||
"jsRules": {},
|
||||
"rules": {
|
||||
"file-name-casing": [true, "kebab-case"],
|
||||
"no-unused-expression": [true, "allow-fast-null-checks"],
|
||||
"interface-name": false,
|
||||
"object-literal-sort-keys": false,
|
||||
"no-object-literal-type-assertion": false,
|
||||
"no-console": true,
|
||||
"member-ordering": false,
|
||||
"no-implicit-dependencies": [true, "dev"],
|
||||
"prefer-conditional-expression": false,
|
||||
"no-submodule-imports": false,
|
||||
"no-default-export": true,
|
||||
"prettier": [true, ".prettierrc.yml"],
|
||||
"no-empty-interface": false,
|
||||
"no-floating-promises": true, // No unawaited or caught promises(This could cause process exit in later version of node)
|
||||
"ordered-imports": [
|
||||
true,
|
||||
{
|
||||
"import-sources-order": "case-insensitive",
|
||||
"named-imports-order": "lowercase-last",
|
||||
"grouped-imports": true,
|
||||
"groups": [
|
||||
{ "match": "^\\..*$", "order": 20 }, // Have relative imports
|
||||
{ "match": ".*", "order": 10 }
|
||||
]
|
||||
}
|
||||
],
|
||||
"ban": [
|
||||
true,
|
||||
{ "name": "fdescribe", "message": "Don't leave focus tests" },
|
||||
{ "name": "fit", "message": "Don't leave focus tests" },
|
||||
{ "name": "ftest", "message": "Don't leave focus tests" },
|
||||
{ "name": ["describe", "only"], "message": "Don't leave focus tests" },
|
||||
{ "name": ["it", "only"], "message": "Don't leave focus tests" },
|
||||
{ "name": ["test", "only"], "message": "Don't leave focus tests" }
|
||||
]
|
||||
},
|
||||
"rulesDirectory": []
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче