Azure
/
ARO-RP
зеркало из https://github.com/Azure/ARO-RP.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

split into http and https listeners, add development mode

This commit is contained in:
Jim Minter 2019-11-18 18:13:18 -06:00
Родитель 347e33f329
Коммит 37397466b0
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 0730CBDA10D1A2D3
9 изменённых файлов: 466 добавлений и 205 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
Dockerfile
Просмотреть файл

@ -2,4 +2,5 @@ FROM registry.access.redhat.com/ubi8/ubi-minimal
COPY rp /usr/local/bin
ENTRYPOINT ["rp"]
EXPOSE 8443/tcp
EXPOSE 8080/tcp
USER 1000

14
cmd/rp/rp.go
Просмотреть файл

@ -3,16 +3,13 @@ package main
import (
"context"
"fmt"
"net"
"net/http"
_ "net/http/pprof"
"os"
"os/signal"
"syscall"
"github.com/sirupsen/logrus"
"github.com/jim-minter/rp/pkg/api"
_ "github.com/jim-minter/rp/pkg/api/v20191231preview"
"github.com/jim-minter/rp/pkg/backend"
"github.com/jim-minter/rp/pkg/database"
@ -39,7 +36,7 @@ func run(ctx context.Context, log *logrus.Entry) error {
}
}
env, err := env.NewEnv(os.Getenv("AZURE_SUBSCRIPTION_ID"), os.Getenv("RESOURCEGROUP"))
env, err := env.NewEnv(ctx, log, os.Getenv("AZURE_SUBSCRIPTION_ID"), os.Getenv("RESOURCEGROUP"))
if err != nil {
return err
}
@ -64,11 +61,6 @@ func run(ctx context.Context, log *logrus.Entry) error {
return err
}
servingCert, err := env.ServingCert(ctx)
if err != nil {
return err
}
authorizer, err := env.FirstPartyAuthorizer(ctx)
if err != nil {
return err
@ -80,14 +72,14 @@ func run(ctx context.Context, log *logrus.Entry) error {
go backend.NewBackend(log.WithField("component", "backend"), authorizer, db, domain).Run(stop)
l, err := net.Listen("tcp", ":8443")
f, err := frontend.NewFrontend(ctx, log.WithField("component", "frontend"), env, db)
if err != nil {
return err
}
log.Print("listening")
go frontend.NewFrontend(log.WithField("component", "frontend"), l, servingCert, db, api.APIs).Run(stop)
f.Run(stop)
<-sigterm
log.Print("received SIGTERM")

17
hack/db/db.go
Просмотреть файл

@ -7,6 +7,7 @@ import (
"os"
uuid "github.com/satori/go.uuid"
"github.com/sirupsen/logrus"
"github.com/ugorji/go/codec"
"github.com/jim-minter/rp/pkg/api"
@ -15,7 +16,7 @@ import (
"github.com/jim-minter/rp/pkg/env"
)
func run(ctx context.Context) error {
func run(ctx context.Context, log *logrus.Entry) error {
for _, key := range []string{
"RESOURCEGROUP",
} {
@ -28,6 +29,11 @@ func run(ctx context.Context) error {
return fmt.Errorf("usage: %s resourceid", os.Args[0])
}
env, err := env.NewEnv(ctx, log, os.Getenv("AZURE_SUBSCRIPTION_ID"), os.Getenv("RESOURCEGROUP"))
if err != nil {
return err
}
databaseAccount, masterKey, err := env.CosmosDB(ctx)
if err != nil {
return err
@ -65,7 +71,14 @@ func run(ctx context.Context) error {
}
func main() {
if err := run(context.Background()); err != nil {
logrus.SetReportCaller(true)
logrus.SetFormatter(&logrus.TextFormatter{
FullTimestamp: true,
DisableLevelTruncation: true,
})
log := logrus.NewEntry(logrus.StandardLogger())
if err := run(context.Background(), log); err != nil {
panic(err)
}
}

52
pkg/env/dev/dev.go поставляемый Normal file
Просмотреть файл

@ -0,0 +1,52 @@
package dev
import (
"context"
"crypto/tls"
"net"
"github.com/sirupsen/logrus"
"github.com/jim-minter/rp/pkg/env/shared"
)
type dev struct {
*shared.Shared
}
func New(ctx context.Context, log *logrus.Entry, subscriptionId, resourceGroup string) (*dev, error) {
var err error
d := &dev{}
d.Shared, err = shared.NewShared(ctx, log, subscriptionId, resourceGroup)
if err != nil {
return nil, err
}
return d, nil
}
func (d *dev) ListenTLS(ctx context.Context) (net.Listener, error) {
key, cert, err := d.GetSecret(ctx, "tls")
if err != nil {
return nil, err
}
// no TLS client cert verification in dev mode, but we'll only listen on
// localhost
return tls.Listen("tcp", "localhost:8443", &tls.Config{
Certificates: []tls.Certificate{
{
Certificate: [][]byte{
cert.Raw,
},
PrivateKey: key,
},
},
})
}
func (d *dev) IsReady() bool {
return true
}

186
pkg/env/env.go поставляемый
Просмотреть файл

@ -2,181 +2,29 @@ package env
import (
"context"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"net"
"os"
"strings"
"github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb"
"github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2018-05-01/dns"
"github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
keyvaultmgmt "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2016-10-01/keyvault"
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/adal"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/sirupsen/logrus"
"github.com/jim-minter/rp/pkg/env/dev"
"github.com/jim-minter/rp/pkg/env/prod"
)
type Env struct {
databaseaccounts documentdb.DatabaseAccountsClient
keyvault keyvault.BaseClient
vaults keyvaultmgmt.VaultsClient
zones dns.ZonesClient
resourceGroup string
type Interface interface {
CosmosDB(ctx context.Context) (string, string, error)
DNS(ctx context.Context) (string, error)
FirstPartyAuthorizer(ctx context.Context) (autorest.Authorizer, error)
IsReady() bool
ListenTLS(ctx context.Context) (net.Listener, error)
}
func NewEnv(subscriptionId, resourceGroup string) (*Env, error) {
authorizer, err := auth.NewAuthorizerFromEnvironment()
if err != nil {
return nil, err
func NewEnv(ctx context.Context, log *logrus.Entry, subscriptionId, resourceGroup string) (Interface, error) {
if strings.ToLower(os.Getenv("RP_MODE")) == "development" {
log.Warn("running in development mode")
return dev.New(ctx, log, subscriptionId, resourceGroup)
}
vaultauthorizer, err := auth.NewAuthorizerFromEnvironmentWithResource("https://vault.azure.net")
if err != nil {
return nil, err
}
e := &Env{
resourceGroup: resourceGroup,
}
e.databaseaccounts = documentdb.NewDatabaseAccountsClient(subscriptionId)
e.keyvault = keyvault.New()
e.vaults = keyvaultmgmt.NewVaultsClient(subscriptionId)
e.zones = dns.NewZonesClient(subscriptionId)
e.databaseaccounts.Authorizer = authorizer
e.keyvault.Authorizer = vaultauthorizer
e.vaults.Authorizer = authorizer
e.zones.Authorizer = authorizer
return e, nil
}
func (e *Env) CosmosDB(ctx context.Context) (string, string, error) {
accts, err := e.databaseaccounts.ListByResourceGroup(ctx, e.resourceGroup)
if err != nil {
return "", "", err
}
if len(*accts.Value) != 1 {
return "", "", fmt.Errorf("found %d database accounts, expected 1", len(*accts.Value))
}
keys, err := e.databaseaccounts.ListKeys(ctx, e.resourceGroup, *(*accts.Value)[0].Name)
if err != nil {
return "", "", err
}
return *(*accts.Value)[0].Name, *keys.PrimaryMasterKey, nil
}
func (e *Env) DNS(ctx context.Context) (string, error) {
page, err := e.zones.ListByResourceGroup(ctx, e.resourceGroup, nil)
if err != nil {
return "", err
}
zones := page.Values()
if len(zones) != 1 {
return "", fmt.Errorf("found at least %d zones, expected 1", len(zones))
}
return *zones[0].Name, nil
}
func (e *Env) getSecret(ctx context.Context, vaultBaseURL, secretName string) (*rsa.PrivateKey, *x509.Certificate, error) {
bundle, err := e.keyvault.GetSecret(ctx, vaultBaseURL, secretName, "")
if err != nil {
return nil, nil, err
}
var key *rsa.PrivateKey
var cert *x509.Certificate
b := []byte(*bundle.Value)
for {
var block *pem.Block
block, b = pem.Decode(b)
if block == nil {
break
}
switch block.Type {
case "PRIVATE KEY":
k, err := x509.ParsePKCS8PrivateKey(block.Bytes)
if err != nil {
return nil, nil, err
}
var ok bool
key, ok = k.(*rsa.PrivateKey)
if !ok {
return nil, nil, errors.New("found unknown private key type in PKCS#8 wrapping")
}
case "CERTIFICATE":
cert, err = x509.ParseCertificate(block.Bytes)
if err != nil {
return nil, nil, err
}
}
}
return key, cert, nil
}
func (e *Env) FirstPartyAuthorizer(ctx context.Context) (autorest.Authorizer, error) {
page, err := e.vaults.ListByResourceGroup(ctx, e.resourceGroup, nil)
if err != nil {
return nil, err
}
vaults := page.Values()
if len(vaults) != 1 {
return nil, fmt.Errorf("found at least %d vaults, expected 1", len(vaults))
}
key, cert, err := e.getSecret(ctx, *vaults[0].Properties.VaultURI, "azure")
if err != nil {
return nil, err
}
oauthConfig, err := adal.NewOAuthConfig(azure.PublicCloud.ActiveDirectoryEndpoint, os.Getenv("AZURE_TENANT_ID"))
if err != nil {
return nil, err
}
sp, err := adal.NewServicePrincipalTokenFromCertificate(*oauthConfig, os.Getenv("AZURE_CLIENT_ID"), cert, key, azure.PublicCloud.ResourceManagerEndpoint)
if err != nil {
return nil, err
}
return autorest.NewBearerAuthorizer(sp), nil
}
func (e *Env) ServingCert(ctx context.Context) (*tls.Certificate, error) {
page, err := e.vaults.ListByResourceGroup(ctx, e.resourceGroup, nil)
if err != nil {
return nil, err
}
vaults := page.Values()
if len(vaults) != 1 {
return nil, fmt.Errorf("found at least %d vaults, expected 1", len(vaults))
}
key, cert, err := e.getSecret(ctx, *vaults[0].Properties.VaultURI, "tls")
if err != nil {
return nil, err
}
return &tls.Certificate{
Certificate: [][]byte{
cert.Raw,
},
PrivateKey: key,
}, nil
return prod.New(ctx, log, subscriptionId, resourceGroup)
}

107
pkg/env/prod/metadata.go поставляемый Normal file
Просмотреть файл

@ -0,0 +1,107 @@
package prod
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
"sync"
"time"
"github.com/sirupsen/logrus"
)
type metadata struct {
ClientCertificates []struct {
Thumbprint string `json:"thumbprint,omitempty"`
NotBefore time.Time `json:"notBefore,omitempty"`
NotAfter time.Time `json:"notAfter,omitempty"`
Certificate []byte `json:"certificate,omitempty"`
} `json:"clientCertificates,omitempty"`
}
type metadataService struct {
log *logrus.Entry
mu sync.RWMutex
m metadata
lastSuccessfulRefresh time.Time
}
func NewMetadataService(log *logrus.Entry) *metadataService {
ms := &metadataService{log: log}
go ms.refresh()
return ms
}
func (ms *metadataService) allowClientCertificate(rawCert []byte) bool {
ms.mu.RLock()
defer ms.mu.RUnlock()
now := time.Now()
for _, c := range ms.m.ClientCertificates {
if c.NotBefore.Before(now) &&
c.NotAfter.After(now) &&
bytes.Equal(c.Certificate, rawCert) {
return true
}
}
return false
}
func (ms *metadataService) refresh() {
t := time.NewTicker(time.Hour)
for {
ms.log.Print("refreshing metadata")
err := ms.refreshOnce()
if err != nil {
ms.log.Warnf("metadata refresh: %v", err)
}
<-t.C
}
}
func (ms *metadataService) refreshOnce() error {
now := time.Now()
resp, err := http.Get("https://management.azure.com:24582/metadata/authentication?api-version=2015-01-01")
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code %q", resp.StatusCode)
}
if strings.SplitN(resp.Header.Get("Content-Type"), ";", 2)[0] != "application/json" {
return fmt.Errorf("unexpected content type %q", resp.Header.Get("Content-Type"))
}
var m *metadata
err = json.NewDecoder(resp.Body).Decode(&m)
if err != nil {
return err
}
ms.mu.Lock()
defer ms.mu.Unlock()
ms.m = *m
ms.lastSuccessfulRefresh = now
return nil
}
func (ms *metadataService) isReady() bool {
ms.mu.RLock()
defer ms.mu.RUnlock()
return time.Now().Add(-6 * time.Hour).Before(ms.lastSuccessfulRefresh)
}

62
pkg/env/prod/prod.go поставляемый Normal file
Просмотреть файл

@ -0,0 +1,62 @@
package prod
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"net"
"github.com/sirupsen/logrus"
"github.com/jim-minter/rp/pkg/env/shared"
)
type prod struct {
*shared.Shared
ms *metadataService
}
func New(ctx context.Context, log *logrus.Entry, subscriptionId, resourceGroup string) (*prod, error) {
var err error
p := &prod{
ms: NewMetadataService(log),
}
p.Shared, err = shared.NewShared(ctx, log, subscriptionId, resourceGroup)
if err != nil {
return nil, err
}
return p, nil
}
func (p *prod) ListenTLS(ctx context.Context) (net.Listener, error) {
key, cert, err := p.GetSecret(ctx, "tls")
if err != nil {
return nil, err
}
return tls.Listen("tcp", ":8443", &tls.Config{
Certificates: []tls.Certificate{
{
Certificate: [][]byte{
cert.Raw,
},
PrivateKey: key,
},
},
ClientAuth: tls.RequestClientCert,
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 || !p.ms.allowClientCertificate(rawCerts[0]) {
return errors.New("invalid certificate")
}
return nil
},
})
}
func (p *prod) IsReady() bool {
return p.ms.isReady()
}

160
pkg/env/shared/shared.go поставляемый Normal file
Просмотреть файл

@ -0,0 +1,160 @@
package shared
import (
"context"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"os"
"github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb"
"github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2018-05-01/dns"
"github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
keyvaultmgmt "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2016-10-01/keyvault"
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/adal"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/sirupsen/logrus"
)
type Shared struct {
databaseaccounts documentdb.DatabaseAccountsClient
keyvault keyvault.BaseClient
vaults keyvaultmgmt.VaultsClient
zones dns.ZonesClient
resourceGroup string
vaultURI string
}
func NewShared(ctx context.Context, log *logrus.Entry, subscriptionId, resourceGroup string) (*Shared, error) {
authorizer, err := auth.NewAuthorizerFromEnvironment()
if err != nil {
return nil, err
}
vaultauthorizer, err := auth.NewAuthorizerFromEnvironmentWithResource("https://vault.azure.net")
if err != nil {
return nil, err
}
s := &Shared{
resourceGroup: resourceGroup,
}
s.databaseaccounts = documentdb.NewDatabaseAccountsClient(subscriptionId)
s.keyvault = keyvault.New()
s.vaults = keyvaultmgmt.NewVaultsClient(subscriptionId)
s.zones = dns.NewZonesClient(subscriptionId)
s.databaseaccounts.Authorizer = authorizer
s.keyvault.Authorizer = vaultauthorizer
s.vaults.Authorizer = authorizer
s.zones.Authorizer = authorizer
page, err := s.vaults.ListByResourceGroup(ctx, s.resourceGroup, nil)
if err != nil {
return nil, err
}
vaults := page.Values()
if len(vaults) != 1 {
return nil, fmt.Errorf("found at least %d vaults, expected 1", len(vaults))
}
s.vaultURI = *vaults[0].Properties.VaultURI
return s, nil
}
func (s *Shared) CosmosDB(ctx context.Context) (string, string, error) {
accts, err := s.databaseaccounts.ListByResourceGroup(ctx, s.resourceGroup)
if err != nil {
return "", "", err
}
if len(*accts.Value) != 1 {
return "", "", fmt.Errorf("found %d database accounts, expected 1", len(*accts.Value))
}
keys, err := s.databaseaccounts.ListKeys(ctx, s.resourceGroup, *(*accts.Value)[0].Name)
if err != nil {
return "", "", err
}
return *(*accts.Value)[0].Name, *keys.PrimaryMasterKey, nil
}
func (s *Shared) DNS(ctx context.Context) (string, error) {
page, err := s.zones.ListByResourceGroup(ctx, s.resourceGroup, nil)
if err != nil {
return "", err
}
zones := page.Values()
if len(zones) != 1 {
return "", fmt.Errorf("found at least %d zones, expected 1", len(zones))
}
return *zones[0].Name, nil
}
func (s *Shared) GetSecret(ctx context.Context, secretName string) (*rsa.PrivateKey, *x509.Certificate, error) {
bundle, err := s.keyvault.GetSecret(ctx, s.vaultURI, secretName, "")
if err != nil {
return nil, nil, err
}
var key *rsa.PrivateKey
var cert *x509.Certificate
b := []byte(*bundle.Value)
for {
var block *pem.Block
block, b = pem.Decode(b)
if block == nil {
break
}
switch block.Type {
case "PRIVATE KEY":
k, err := x509.ParsePKCS8PrivateKey(block.Bytes)
if err != nil {
return nil, nil, err
}
var ok bool
key, ok = k.(*rsa.PrivateKey)
if !ok {
return nil, nil, errors.New("found unknown private key type in PKCS#8 wrapping")
}
case "CERTIFICATE":
cert, err = x509.ParseCertificate(block.Bytes)
if err != nil {
return nil, nil, err
}
}
}
return key, cert, nil
}
func (s *Shared) FirstPartyAuthorizer(ctx context.Context) (autorest.Authorizer, error) {
key, cert, err := s.GetSecret(ctx, "azure")
if err != nil {
return nil, err
}
oauthConfig, err := adal.NewOAuthConfig(azure.PublicCloud.ActiveDirectoryEndpoint, os.Getenv("AZURE_TENANT_ID"))
if err != nil {
return nil, err
}
sp, err := adal.NewServicePrincipalTokenFromCertificate(*oauthConfig, os.Getenv("AZURE_CLIENT_ID"), cert, key, azure.PublicCloud.ResourceManagerEndpoint)
if err != nil {
return nil, err
}
return autorest.NewBearerAuthorizer(sp), nil
}

72
pkg/frontend/frontend.go
Просмотреть файл

@ -1,7 +1,7 @@
package frontend
import (
"crypto/tls"
"context"
"net"
"net/http"
"sync/atomic"
@ -11,6 +11,7 @@ import (
"github.com/jim-minter/rp/pkg/api"
"github.com/jim-minter/rp/pkg/database"
"github.com/jim-minter/rp/pkg/env"
)
const (
@ -41,14 +42,14 @@ func validateProvisioningState(state api.ProvisioningState, allowedStates ...api
type frontend struct {
baseLog *logrus.Entry
env env.Interface
db database.OpenShiftClusters
apis map[api.APIVersionType]func(*api.OpenShiftCluster) api.External
db database.OpenShiftClusters
l net.Listener
cert *tls.Certificate
tlsl net.Listener
healthy atomic.Value
ready atomic.Value
}
// Runnable represents a runnable object
@ -57,33 +58,50 @@ type Runnable interface {
}
// NewFrontend returns a new runnable frontend
func NewFrontend(baseLog *logrus.Entry, l net.Listener, cert *tls.Certificate, db database.OpenShiftClusters, apis map[api.APIVersionType]func(*api.OpenShiftCluster) api.External) Runnable {
func NewFrontend(ctx context.Context, baseLog *logrus.Entry, env env.Interface, db database.OpenShiftClusters) (Runnable, error) {
f := &frontend{
baseLog: baseLog,
env: env,
db: db,
apis: apis,
l: l,
cert: cert,
}
f.healthy.Store(true)
var err error
f.l, err = net.Listen("tcp", ":8080")
if err != nil {
return nil, err
}
return f
f.tlsl, err = f.env.ListenTLS(ctx)
if err != nil {
return nil, err
}
f.ready.Store(true)
return f, nil
}
func (f *frontend) health(w http.ResponseWriter, r *http.Request) {
if f.healthy.Load().(bool) {
http.Error(w, "", http.StatusOK)
func (f *frontend) getReady(w http.ResponseWriter, r *http.Request) {
if f.ready.Load().(bool) && f.env.IsReady() {
http.Error(w, http.StatusText(http.StatusOK), http.StatusOK)
} else {
http.Error(w, "", http.StatusInternalServerError)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
}
}
func (f *frontend) Run(stop <-chan struct{}) {
// unauthenticatedRouter returns the router which is served via unauthenticated
// HTTP
func (f *frontend) unauthenticatedRouter() *mux.Router {
r := mux.NewRouter()
r.Path("/healthz/ready").Methods(http.MethodGet).HandlerFunc(f.getReady)
return r
}
// authenticatedRouter returns the router which is served via TLS and protected
// by client certificate authentication
func (f *frontend) authenticatedRouter() *mux.Router {
r := mux.NewRouter()
r.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
r.Path("/health").Methods(http.MethodGet).HandlerFunc(f.health)
s := r.
Path("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}").
@ -120,13 +138,21 @@ func (f *frontend) Run(stop <-chan struct{}) {
s.Use(f.middleware)
s.Methods(http.MethodPost).HandlerFunc(f.postOpenShiftClusterCredentials)
return r
}
func (f *frontend) Run(stop <-chan struct{}) {
go func() {
<-stop
f.baseLog.Println("marking frontend unhealthy")
f.healthy.Store(false)
f.baseLog.Print("marking frontend not ready")
f.ready.Store(false)
}()
l := tls.NewListener(f.l, &tls.Config{Certificates: []tls.Certificate{*f.cert}})
err := http.Serve(l, r)
go func() {
err := http.Serve(f.l, f.unauthenticatedRouter())
f.baseLog.Error(err)
}()
err := http.Serve(f.tlsl, f.authenticatedRouter())
f.baseLog.Error(err)
}