зеркало из https://github.com/Azure/ARO-RP.git
Fix bad previous bad rebase
This commit is contained in:
Ellis Johnson
Родитель
09e0d6c2ab
Коммит
b57d647632
|
|
@ -27,7 +27,7 @@ jobs:
|
|||
uses: golangci/golangci-lint-action@v3
|
||||
with:
|
||||
# Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
|
||||
version: v1.45.2
|
||||
version: v1.49.0
|
||||
args: -v --timeout 15m
|
||||
# Optional: working directory, useful for monorepos
|
||||
#working-directory: pkg
|
||||
|
|
|
|||
|
|
@ -43,6 +43,9 @@ jobs:
|
|||
az account set -s $AZURE_SUBSCRIPTION_ID
|
||||
|
||||
set -x
|
||||
|
||||
export PRIVATE_CLUSTER=true
|
||||
|
||||
. ./hack/e2e/run-rp-and-e2e.sh
|
||||
trap 'set +e; kill_rp; kill_portal; clean_e2e_db; kill_vpn' EXIT
|
||||
|
||||
|
|
|
|||
|
|
@ -216,6 +216,34 @@
|
|||
curl -X GET -k "https://localhost:8443/admin/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCEGROUP/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER/kubernetespodlogs?podname=$POD&namespace=$NAMESPACE&container=$CONTAINER"
|
||||
```
|
||||
|
||||
## OpenShift Version
|
||||
|
||||
* We have a cosmos container which contains supported installable OCP versions, more information on the definition in `pkg/api/openshiftversion.go`.
|
||||
|
||||
* Admin - List OpenShift installation versions
|
||||
```bash
|
||||
curl -X GET -k "https://localhost:8443/admin/versions"
|
||||
```
|
||||
|
||||
* Admin - Put a new OpenShift installation version
|
||||
```bash
|
||||
curl -X PUT -k "https://localhost:8443/admin/versions" --header "Content-Type: application/json" -d '{ "properties": { "version": "4.10.0", "enabled": true, "openShiftPullspec": "test.com/a:b", "installerPullspec": "test.com/a:b" }}'
|
||||
```
|
||||
|
||||
* List the enabled OpenShift installation versions within a region
|
||||
```bash
|
||||
curl -X GET -k "https://localhost:8443/subscriptions/$AZURE_SUBSCRIPTION_ID/providers/Microsoft.RedHatOpenShift/locations/$LOCATION/listinstallversions?api-version=2022-09-04"
|
||||
```
|
||||
|
||||
## OpenShift Cluster Manager (OCM) Configuration API Actions
|
||||
|
||||
* Create a new OCM configuration
|
||||
* You can find example payloads in the projects `./hack/ocm` folder.
|
||||
|
||||
```bash
|
||||
curl -X PUT -k "https://localhost:8443/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCEGROUP/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER/syncsets/mySyncSet?api-version=2022-09-04" --header "Content-Type: application/json" -d @./hack/ocm/syncset.b64
|
||||
|
||||
|
||||
## Debugging OpenShift Cluster
|
||||
|
||||
* SSH to the bootstrap node:
|
||||
|
|
|
|||
6
go.mod
6
go.mod
|
|
@ -38,7 +38,7 @@ require (
|
|||
github.com/gorilla/sessions v1.2.1
|
||||
github.com/jewzaam/go-cosmosdb v0.0.0-20220315232836-282b67c5b234
|
||||
github.com/jstemmer/go-junit-report v0.9.1
|
||||
github.com/onsi/ginkgo v1.16.5
|
||||
github.com/onsi/ginkgo/v2 v2.1.3
|
||||
github.com/onsi/gomega v1.19.0
|
||||
github.com/openshift/api v3.9.1-0.20191111211345-a27ff30ebf09+incompatible
|
||||
github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a
|
||||
|
|
@ -156,6 +156,7 @@ require (
|
|||
github.com/go-playground/universal-translator v0.18.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.10.1 // indirect
|
||||
github.com/go-stack/stack v1.8.1 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
|
||||
github.com/go-toolsmith/astcast v1.0.0 // indirect
|
||||
github.com/go-toolsmith/astcopy v1.0.0 // indirect
|
||||
github.com/go-toolsmith/astequal v1.0.0 // indirect
|
||||
|
|
@ -182,6 +183,7 @@ require (
|
|||
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
|
||||
github.com/google/btree v1.0.1 // indirect
|
||||
github.com/google/gofuzz v1.2.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
|
||||
github.com/google/renameio v1.0.1 // indirect
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
|
|
@ -253,7 +255,6 @@ require (
|
|||
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
|
||||
github.com/nishanths/exhaustive v0.2.3 // indirect
|
||||
github.com/nishanths/predeclared v0.2.1 // indirect
|
||||
github.com/nxadm/tail v1.4.8 // indirect
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
|
|
@ -335,7 +336,6 @@ require (
|
|||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/ini.v1 v1.66.4 // indirect
|
||||
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||
honnef.co/go/tools v0.2.1 // indirect
|
||||
|
|
|
|||
|
|
@ -60,6 +60,12 @@ func addRequiredResources(requiredResources map[string]int, vmSize api.VMSize, c
|
|||
api.VMSizeStandardL32sV2: {CoreCount: 32, Family: "standardLsv2Family"},
|
||||
api.VMSizeStandardL48sV2: {CoreCount: 48, Family: "standardLsv2Family"},
|
||||
api.VMSizeStandardL64sV2: {CoreCount: 64, Family: "standardLsv2Family"},
|
||||
|
||||
// GPU nodes
|
||||
api.VMSizeStandardNC4asT4V3: {CoreCount: 4, Family: "Standard_NC4as_T4_v3"},
|
||||
api.VMSizeStandardNC8asT4V3: {CoreCount: 8, Family: "Standard_NC8as_T4_v3"},
|
||||
api.VMSizeStandardNC16asT4V3: {CoreCount: 16, Family: "Standard_NC16as_T4_v3"},
|
||||
api.VMSizeStandardNC64asT4V3: {CoreCount: 64, Family: "Standard_NC64as_T4_v3"},
|
||||
}
|
||||
|
||||
vm, ok := vmTypesMap[vmSize]
|
||||
|
|
|
|||
|
|
@ -74,6 +74,11 @@ var supportedWorkerVMSizes = map[api.VMSize]bool{
|
|||
api.VMSizeStandardL32sV2: true,
|
||||
api.VMSizeStandardL48sV2: true,
|
||||
api.VMSizeStandardL64sV2: true,
|
||||
// GPU
|
||||
api.VMSizeStandardNC4asT4V3: true,
|
||||
api.VMSizeStandardNC8asT4V3: true,
|
||||
api.VMSizeStandardNC16asT4V3: true,
|
||||
api.VMSizeStandardNC64asT4V3: true,
|
||||
}
|
||||
|
||||
func DiskSizeIsValid(sizeGB int) bool {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"strings"
|
||||
|
||||
mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
|
||||
mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
|
||||
"github.com/Azure/go-autorest/autorest"
|
||||
"github.com/Azure/go-autorest/autorest/azure"
|
||||
"github.com/Azure/go-autorest/autorest/to"
|
||||
|
|
@ -41,14 +42,19 @@ func (m *manager) ensureInfraID(ctx context.Context) (err error) {
|
|||
return err
|
||||
}
|
||||
|
||||
func (m *manager) ensureResourceGroup(ctx context.Context) error {
|
||||
func (m *manager) ensureResourceGroup(ctx context.Context) (err error) {
|
||||
resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/')
|
||||
group := mgmtfeatures.ResourceGroup{}
|
||||
|
||||
// The FPSP's role definition does not have read on a resource group
|
||||
// if the resource group does not exist.
|
||||
// Retain the existing resource group configuration (such as tags) if it exists
|
||||
group, err := m.resourceGroups.Get(ctx, resourceGroup)
|
||||
if err != nil {
|
||||
if detailedErr, ok := err.(autorest.DetailedError); !ok || detailedErr.StatusCode != http.StatusNotFound {
|
||||
return err
|
||||
if m.doc.OpenShiftCluster.Properties.ProvisioningState != api.ProvisioningStateCreating {
|
||||
group, err = m.resourceGroups.Get(ctx, resourceGroup)
|
||||
if err != nil {
|
||||
if detailedErr, ok := err.(autorest.DetailedError); !ok || detailedErr.StatusCode != http.StatusNotFound {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -81,6 +87,16 @@ func (m *manager) ensureResourceGroup(ctx context.Context) error {
|
|||
serviceError = requestErr.ServiceError
|
||||
}
|
||||
|
||||
if serviceError != nil && serviceError.Code == "ResourceGroupManagedByMismatch" {
|
||||
return &api.CloudError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
CloudErrorBody: &api.CloudErrorBody{
|
||||
Code: api.CloudErrorCodeClusterResourceGroupAlreadyExists,
|
||||
Message: "Resource group " + m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID +
|
||||
" must not already exist.",
|
||||
},
|
||||
}
|
||||
}
|
||||
if serviceError != nil && serviceError.Code == "RequestDisallowedByPolicy" {
|
||||
// if request was disallowed by policy, inform user so they can take appropriate action
|
||||
b, _ := json.Marshal(serviceError)
|
||||
|
|
|
|||
|
|
@ -118,7 +118,6 @@ func (m *manager) adminUpdate() []steps.Step {
|
|||
// Update the ARO Operator
|
||||
if isEverything || isOperator {
|
||||
toRun = append(toRun,
|
||||
steps.Action(m.initializeOperatorDeployer), // depends on kube clients
|
||||
steps.Action(m.ensureAROOperator),
|
||||
steps.Condition(m.aroDeploymentReady, 20*time.Minute, true),
|
||||
steps.Condition(m.ensureAROOperatorRunningDesiredVersion, 5*time.Minute, true),
|
||||
|
|
|
|||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -263,6 +263,42 @@ for attempt in {1..5}; do
|
|||
if [[ ${attempt} -lt 5 ]]; then sleep 10; else exit 1; fi
|
||||
done
|
||||
|
||||
cat >/etc/logrotate.conf <<'EOF'
|
||||
# see "man logrotate" for details
|
||||
# rotate log files weekly
|
||||
weekly
|
||||
|
||||
# keep 2 weeks worth of backlogs
|
||||
rotate 2
|
||||
|
||||
# create new (empty) log files after rotating old ones
|
||||
create
|
||||
|
||||
# use date as a suffix of the rotated file
|
||||
dateext
|
||||
|
||||
# uncomment this if you want your log files compressed
|
||||
compress
|
||||
|
||||
# RPM packages drop log rotation information into this directory
|
||||
include /etc/logrotate.d
|
||||
|
||||
# no packages own wtmp and btmp -- we'll rotate them here
|
||||
/var/log/wtmp {
|
||||
monthly
|
||||
create 0664 root utmp
|
||||
minsize 1M
|
||||
rotate 1
|
||||
}
|
||||
|
||||
/var/log/btmp {
|
||||
missingok
|
||||
monthly
|
||||
create 0600 root utmp
|
||||
rotate 1
|
||||
}
|
||||
EOF
|
||||
|
||||
cat >/etc/yum.repos.d/azure.repo <<'EOF'
|
||||
[azure-cli]
|
||||
name=azure-cli
|
||||
|
|
|
|||
|
|
@ -538,6 +538,42 @@ for attempt in {1..5}; do
|
|||
if [[ ${attempt} -lt 5 ]]; then sleep 10; else exit 1; fi
|
||||
done
|
||||
|
||||
cat >/etc/logrotate.conf <<'EOF'
|
||||
# see "man logrotate" for details
|
||||
# rotate log files weekly
|
||||
weekly
|
||||
|
||||
# keep 2 weeks worth of backlogs
|
||||
rotate 2
|
||||
|
||||
# create new (empty) log files after rotating old ones
|
||||
create
|
||||
|
||||
# use date as a suffix of the rotated file
|
||||
dateext
|
||||
|
||||
# uncomment this if you want your log files compressed
|
||||
compress
|
||||
|
||||
# RPM packages drop log rotation information into this directory
|
||||
include /etc/logrotate.d
|
||||
|
||||
# no packages own wtmp and btmp -- we'll rotate them here
|
||||
/var/log/wtmp {
|
||||
monthly
|
||||
create 0664 root utmp
|
||||
minsize 1M
|
||||
rotate 1
|
||||
}
|
||||
|
||||
/var/log/btmp {
|
||||
missingok
|
||||
monthly
|
||||
create 0600 root utmp
|
||||
rotate 1
|
||||
}
|
||||
EOF
|
||||
|
||||
cat >/etc/yum.repos.d/azure.repo <<'EOF'
|
||||
[azure-cli]
|
||||
name=azure-cli
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
|
|
@ -30,6 +31,7 @@ import (
|
|||
"github.com/Azure/ARO-RP/pkg/util/encryption"
|
||||
"github.com/Azure/ARO-RP/pkg/util/heartbeat"
|
||||
"github.com/Azure/ARO-RP/pkg/util/recover"
|
||||
"github.com/Azure/ARO-RP/pkg/util/version"
|
||||
)
|
||||
|
||||
type statusCodeError int
|
||||
|
|
@ -49,12 +51,18 @@ type frontend struct {
|
|||
baseLog *logrus.Entry
|
||||
env env.Interface
|
||||
|
||||
dbAsyncOperations database.AsyncOperations
|
||||
dbOpenShiftClusters database.OpenShiftClusters
|
||||
dbSubscriptions database.Subscriptions
|
||||
dbOpenShiftVersions database.OpenShiftVersions
|
||||
dbAsyncOperations database.AsyncOperations
|
||||
dbClusterManagerConfiguration database.ClusterManagerConfigurations
|
||||
dbOpenShiftClusters database.OpenShiftClusters
|
||||
dbSubscriptions database.Subscriptions
|
||||
dbOpenShiftVersions database.OpenShiftVersions
|
||||
|
||||
enabledOcpVersions map[string]*api.OpenShiftVersion
|
||||
apis map[string]*api.Version
|
||||
|
||||
lastChangefeed atomic.Value //time.Time
|
||||
mu sync.RWMutex
|
||||
|
||||
apis map[string]*api.Version
|
||||
m metrics.Emitter
|
||||
aead encryption.AEAD
|
||||
|
||||
|
|
@ -72,8 +80,9 @@ type frontend struct {
|
|||
ready atomic.Value
|
||||
|
||||
// these helps us to test and mock easier
|
||||
now func() time.Time
|
||||
systemDataEnricher func(*api.OpenShiftClusterDocument, *api.SystemData)
|
||||
now func() time.Time
|
||||
systemDataClusterDocEnricher func(*api.OpenShiftClusterDocument, *api.SystemData)
|
||||
systemDataClusterManagerEnricher func(*api.ClusterManagerConfigurationDocument, *api.SystemData)
|
||||
}
|
||||
|
||||
// Runnable represents a runnable object
|
||||
|
|
@ -87,6 +96,7 @@ func NewFrontend(ctx context.Context,
|
|||
baseLog *logrus.Entry,
|
||||
_env env.Interface,
|
||||
dbAsyncOperations database.AsyncOperations,
|
||||
dbClusterManagerConfiguration database.ClusterManagerConfigurations,
|
||||
dbOpenShiftClusters database.OpenShiftClusters,
|
||||
dbSubscriptions database.Subscriptions,
|
||||
dbOpenShiftVersions database.OpenShiftVersions,
|
||||
|
|
@ -97,26 +107,38 @@ func NewFrontend(ctx context.Context,
|
|||
azureActionsFactory azureActionsFactory,
|
||||
ocEnricherFactory ocEnricherFactory) (Runnable, error) {
|
||||
f := &frontend{
|
||||
auditLog: auditLog,
|
||||
baseLog: baseLog,
|
||||
env: _env,
|
||||
dbAsyncOperations: dbAsyncOperations,
|
||||
dbOpenShiftClusters: dbOpenShiftClusters,
|
||||
dbSubscriptions: dbSubscriptions,
|
||||
dbOpenShiftVersions: dbOpenShiftVersions,
|
||||
apis: apis,
|
||||
m: m,
|
||||
aead: aead,
|
||||
kubeActionsFactory: kubeActionsFactory,
|
||||
azureActionsFactory: azureActionsFactory,
|
||||
ocEnricherFactory: ocEnricherFactory,
|
||||
auditLog: auditLog,
|
||||
baseLog: baseLog,
|
||||
env: _env,
|
||||
dbAsyncOperations: dbAsyncOperations,
|
||||
dbClusterManagerConfiguration: dbClusterManagerConfiguration,
|
||||
dbOpenShiftClusters: dbOpenShiftClusters,
|
||||
dbSubscriptions: dbSubscriptions,
|
||||
dbOpenShiftVersions: dbOpenShiftVersions,
|
||||
apis: apis,
|
||||
m: m,
|
||||
aead: aead,
|
||||
kubeActionsFactory: kubeActionsFactory,
|
||||
azureActionsFactory: azureActionsFactory,
|
||||
ocEnricherFactory: ocEnricherFactory,
|
||||
|
||||
// add default installation version so it's always supported
|
||||
enabledOcpVersions: map[string]*api.OpenShiftVersion{
|
||||
version.InstallStream.Version.String(): {
|
||||
Properties: api.OpenShiftVersionProperties{
|
||||
Version: version.InstallStream.Version.String(),
|
||||
Enabled: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
bucketAllocator: &bucket.Random{},
|
||||
|
||||
startTime: time.Now(),
|
||||
|
||||
now: time.Now,
|
||||
systemDataEnricher: enrichSystemData,
|
||||
now: time.Now,
|
||||
systemDataClusterDocEnricher: enrichClusterSystemData,
|
||||
systemDataClusterManagerEnricher: enrichClusterManagerSystemData,
|
||||
}
|
||||
|
||||
l, err := f.env.Listen()
|
||||
|
|
@ -179,6 +201,18 @@ func (f *frontend) authenticatedRoutes(r *mux.Router) {
|
|||
s.Methods(http.MethodPatch).HandlerFunc(f.putOrPatchOpenShiftCluster).Name("putOrPatchOpenShiftCluster")
|
||||
s.Methods(http.MethodPut).HandlerFunc(f.putOrPatchOpenShiftCluster).Name("putOrPatchOpenShiftCluster")
|
||||
|
||||
if f.env.FeatureIsSet(env.FeatureEnableOCMEndpoints) {
|
||||
s = r.
|
||||
Path("/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/{ocmResourceType}/{ocmResourceName}").
|
||||
Queries("api-version", "{api-version}").
|
||||
Subrouter()
|
||||
|
||||
s.Methods(http.MethodDelete).HandlerFunc(f.deleteClusterManagerConfiguration).Name("deleteClusterManagerConfiguration")
|
||||
s.Methods(http.MethodGet).HandlerFunc(f.getClusterManagerConfiguration).Name("getClusterManagerConfiguration")
|
||||
s.Methods(http.MethodPatch).HandlerFunc(f.putOrPatchClusterManagerConfiguration).Name("putOrPatchClusterManagerConfiguration")
|
||||
s.Methods(http.MethodPut).HandlerFunc(f.putOrPatchClusterManagerConfiguration).Name("putOrPatchClusterManagerConfiguration")
|
||||
}
|
||||
|
||||
s = r.
|
||||
Path("/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}").
|
||||
Queries("api-version", "{api-version}").
|
||||
|
|
@ -237,6 +271,12 @@ func (f *frontend) authenticatedRoutes(r *mux.Router) {
|
|||
s.Methods(http.MethodPost).HandlerFunc(f.postAdminKubernetesObjects).Name("postAdminKubernetesObjects")
|
||||
s.Methods(http.MethodDelete).HandlerFunc(f.deleteAdminKubernetesObjects).Name("deleteAdminKubernetesObjects")
|
||||
|
||||
s = r.
|
||||
Path("/admin/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/approvecsr").
|
||||
Subrouter()
|
||||
|
||||
s.Methods(http.MethodPost).HandlerFunc(f.postAdminOpenShiftClusterApproveCSR).Name("postAdminOpenShiftClusterApproveCSR")
|
||||
|
||||
// Pod logs
|
||||
s = r.
|
||||
Path("/admin/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/kubernetespodlogs").
|
||||
|
|
@ -367,6 +407,7 @@ func (f *frontend) setupRouter() *mux.Router {
|
|||
|
||||
func (f *frontend) Run(ctx context.Context, stop <-chan struct{}, done chan<- struct{}) {
|
||||
defer recover.Panic(f.baseLog)
|
||||
go f.changefeed(ctx)
|
||||
|
||||
if stop != nil {
|
||||
go func() {
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ type Operator interface {
|
|||
CreateOrUpdate(context.Context) error
|
||||
IsReady(context.Context) (bool, error)
|
||||
IsRunningDesiredVersion(context.Context) (bool, error)
|
||||
RenewMDSDCertificate(context.Context) error
|
||||
}
|
||||
|
||||
type operator struct {
|
||||
|
|
@ -364,6 +365,31 @@ func (o *operator) CreateOrUpdate(ctx context.Context) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (o *operator) RenewMDSDCertificate(ctx context.Context) error {
|
||||
key, cert := o.env.ClusterGenevaLoggingSecret()
|
||||
gcsKeyBytes, err := utiltls.PrivateKeyAsBytes(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gcsCertBytes, err := utiltls.CertAsBytes(cert)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s, err := o.kubernetescli.CoreV1().Secrets(pkgoperator.Namespace).Get(ctx, pkgoperator.SecretName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.Data["gcscert.pem"] = gcsCertBytes
|
||||
s.Data["gcskey.pem"] = gcsKeyBytes
|
||||
|
||||
_, err = o.kubernetescli.CoreV1().Secrets(pkgoperator.Namespace).Update(ctx, s, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *operator) IsReady(ctx context.Context) (bool, error) {
|
||||
ok, err := ready.CheckDeploymentIsReady(ctx, o.kubernetescli.AppsV1().Deployments(pkgoperator.Namespace), "aro-operator-master")()
|
||||
if !ok || err != nil {
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import (
|
|||
mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization"
|
||||
mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
|
||||
"github.com/Azure/go-autorest/autorest"
|
||||
"github.com/Azure/go-autorest/autorest/azure"
|
||||
"github.com/Azure/go-autorest/autorest/azure/auth"
|
||||
"github.com/Azure/go-autorest/autorest/to"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
|
@ -46,9 +47,10 @@ import (
|
|||
)
|
||||
|
||||
type Cluster struct {
|
||||
log *logrus.Entry
|
||||
env env.Core
|
||||
ci bool
|
||||
log *logrus.Entry
|
||||
env env.Core
|
||||
ci bool
|
||||
ciParentVnet string
|
||||
|
||||
deployments features.DeploymentsClient
|
||||
groups features.ResourceGroupsClient
|
||||
|
|
@ -63,6 +65,7 @@ type Cluster struct {
|
|||
routetables network.RouteTablesClient
|
||||
roleassignments authorization.RoleAssignmentsClient
|
||||
peerings network.VirtualNetworkPeeringsClient
|
||||
ciParentVnetPeerings network.VirtualNetworkPeeringsClient
|
||||
vaultsClient keyvaultclient.VaultsClient
|
||||
}
|
||||
|
||||
|
|
@ -121,6 +124,18 @@ func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) {
|
|||
vaultsClient: keyvaultclient.NewVaultsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
|
||||
}
|
||||
|
||||
if ci && env.IsLocalDevelopmentMode() {
|
||||
// Only peer if CI=true and RP_MODE=development
|
||||
c.ciParentVnet = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vpn-vnet", c.env.SubscriptionID(), c.env.ResourceGroup())
|
||||
|
||||
r, err := azure.ParseResourceID(c.ciParentVnet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.ciParentVnetPeerings = network.NewVirtualNetworkPeeringsClient(environment.Environment(), r.SubscriptionID, authorizer)
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
|
|
@ -308,6 +323,14 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if env.IsLocalDevelopmentMode() {
|
||||
c.log.Info("peering subnets to CI infra")
|
||||
err = c.peerSubnetsToCI(ctx, vnetResourceGroup, clusterName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
c.log.Info("done")
|
||||
|
|
@ -315,13 +338,15 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str
|
|||
}
|
||||
|
||||
func (c *Cluster) generateSubnets() (vnetPrefix string, masterSubnet string, workerSubnet string) {
|
||||
// pick a random /23 in the range [10.0.2.0, 10.128.0.0). 10.0.0.0 is used
|
||||
// by dev-vnet to host CI; 10.128.0.0+ is used for pods.
|
||||
// pick a random 23 in range [10.3.0.0, 10.127.255.0]
|
||||
// 10.0.0.0 is used by dev-vnet to host CI
|
||||
// 10.1.0.0 is used by rp-vnet to host Proxy VM
|
||||
// 10.2.0.0 is used by dev-vpn-vnet to host VirtualNetworkGateway
|
||||
var x, y int
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
for x == 0 && y == 0 {
|
||||
x, y = rand.Intn(128), 2*rand.Intn(128)
|
||||
x, y = rand.Intn((124))+3, 2*rand.Intn(128)
|
||||
}
|
||||
|
||||
vnetPrefix = fmt.Sprintf("10.%d.%d.0/23", x, y)
|
||||
masterSubnet = fmt.Sprintf("10.%d.%d.0/24", x, y)
|
||||
workerSubnet = fmt.Sprintf("10.%d.%d.0/24", x, y+1)
|
||||
|
|
@ -359,6 +384,16 @@ func (c *Cluster) Delete(ctx context.Context, vnetResourceGroup, clusterName str
|
|||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
// Only delete peering if CI=true and RP_MODE=development
|
||||
if env.IsLocalDevelopmentMode() {
|
||||
r, err := azure.ParseResourceID(c.ciParentVnet)
|
||||
if err == nil {
|
||||
err = c.ciParentVnetPeerings.DeleteAndWait(ctx, r.ResourceGroup, r.ResourceName, vnetResourceGroup+"-peer")
|
||||
}
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Deleting the deployment does not clean up the associated resources
|
||||
c.log.Info("deleting deployment")
|
||||
|
|
@ -454,7 +489,7 @@ func (c *Cluster) createCluster(ctx context.Context, vnetResourceGroup, clusterN
|
|||
oc.Properties.WorkerProfiles[0].VMSize = api.VMSizeStandardD2sV3
|
||||
}
|
||||
|
||||
ext := api.APIs[v20220401.APIVersion].OpenShiftClusterConverter().ToExternal(&oc)
|
||||
ext := api.APIs[v20220401.APIVersion].OpenShiftClusterConverter.ToExternal(&oc)
|
||||
data, err := json.Marshal(ext)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -574,3 +609,41 @@ func (c *Cluster) deleteRoleAssignments(ctx context.Context, vnetResourceGroup,
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) peerSubnetsToCI(ctx context.Context, vnetResourceGroup, clusterName string) error {
|
||||
cluster := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vnet", c.env.SubscriptionID(), vnetResourceGroup)
|
||||
|
||||
r, err := azure.ParseResourceID(c.ciParentVnet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
clusterProp := &mgmtnetwork.VirtualNetworkPeeringPropertiesFormat{
|
||||
RemoteVirtualNetwork: &mgmtnetwork.SubResource{
|
||||
ID: &c.ciParentVnet,
|
||||
},
|
||||
AllowVirtualNetworkAccess: to.BoolPtr(true),
|
||||
AllowForwardedTraffic: to.BoolPtr(true),
|
||||
UseRemoteGateways: to.BoolPtr(true),
|
||||
}
|
||||
rpProp := &mgmtnetwork.VirtualNetworkPeeringPropertiesFormat{
|
||||
RemoteVirtualNetwork: &mgmtnetwork.SubResource{
|
||||
ID: &cluster,
|
||||
},
|
||||
AllowVirtualNetworkAccess: to.BoolPtr(true),
|
||||
AllowForwardedTraffic: to.BoolPtr(true),
|
||||
AllowGatewayTransit: to.BoolPtr(true),
|
||||
}
|
||||
|
||||
err = c.peerings.CreateOrUpdateAndWait(ctx, vnetResourceGroup, "dev-vnet", r.ResourceGroup+"-peer", mgmtnetwork.VirtualNetworkPeering{VirtualNetworkPeeringPropertiesFormat: clusterProp})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = c.ciParentVnetPeerings.CreateOrUpdateAndWait(ctx, r.ResourceGroup, r.ResourceName, vnetResourceGroup+"-peer", mgmtnetwork.VirtualNetworkPeering{VirtualNetworkPeeringPropertiesFormat: rpProp})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -7,7 +7,7 @@ import (
|
|||
"os"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo" //nolint
|
||||
. "github.com/onsi/ginkgo/v2" //nolint
|
||||
. "github.com/onsi/gomega" //nolint
|
||||
. "github.com/serge1peshcoff/selenium-go-conditions" //nolint
|
||||
. "github.com/tebeka/selenium" //nolint
|
||||
|
|
|
|||
|
|
@ -5,15 +5,13 @@ package e2e
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
|
||||
|
|
@ -30,8 +28,8 @@ import (
|
|||
"k8s.io/client-go/util/retry"
|
||||
|
||||
arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
|
||||
imageController "github.com/Azure/ARO-RP/pkg/operator/controllers/imageconfig"
|
||||
"github.com/Azure/ARO-RP/pkg/operator/controllers/monitoring"
|
||||
"github.com/Azure/ARO-RP/pkg/util/azureclient"
|
||||
"github.com/Azure/ARO-RP/pkg/util/conditions"
|
||||
"github.com/Azure/ARO-RP/pkg/util/ready"
|
||||
"github.com/Azure/ARO-RP/pkg/util/subnet"
|
||||
|
|
@ -79,7 +77,7 @@ func dumpEvents(ctx context.Context, namespace string) error {
|
|||
var _ = Describe("ARO Operator - Internet checking", func() {
|
||||
var originalURLs []string
|
||||
BeforeEach(func() {
|
||||
// save the originalURLs
|
||||
By("saving the original URLs")
|
||||
co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(context.Background(), "cluster", metav1.GetOptions{})
|
||||
if kerrors.IsNotFound(err) {
|
||||
Skip("skipping tests as aro-operator is not deployed")
|
||||
|
|
@ -89,7 +87,7 @@ var _ = Describe("ARO Operator - Internet checking", func() {
|
|||
originalURLs = co.Spec.InternetChecker.URLs
|
||||
})
|
||||
AfterEach(func() {
|
||||
// set the URLs back again
|
||||
By("restoring the original URLs")
|
||||
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(context.Background(), "cluster", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
|
|
@ -101,20 +99,20 @@ var _ = Describe("ARO Operator - Internet checking", func() {
|
|||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
Specify("the InternetReachable default list should all be reachable", func() {
|
||||
It("sets InternetReachableFromMaster to true when the default URL is reachable from master nodes", func() {
|
||||
co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(context.Background(), "cluster", metav1.GetOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(conditions.IsTrue(co.Status.Conditions, arov1alpha1.InternetReachableFromMaster)).To(BeTrue())
|
||||
})
|
||||
|
||||
Specify("the InternetReachable default list should all be reachable from worker", func() {
|
||||
It("sets InternetReachableFromWorker to true when the default URL is reachable from worker nodes", func() {
|
||||
co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(context.Background(), "cluster", metav1.GetOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(conditions.IsTrue(co.Status.Conditions, arov1alpha1.InternetReachableFromWorker)).To(BeTrue())
|
||||
})
|
||||
|
||||
Specify("custom invalid site shows not InternetReachable", func() {
|
||||
// set an unreachable URL
|
||||
It("sets InternetReachableFromMaster and InternetReachableFromWorker to false when URL is not reachable", func() {
|
||||
By("setting a deliberately unreachable URL")
|
||||
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(context.Background(), "cluster", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
|
|
@ -126,7 +124,7 @@ var _ = Describe("ARO Operator - Internet checking", func() {
|
|||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// confirm the conditions are correct
|
||||
By("waiting for the expected conditions to be set")
|
||||
err = wait.PollImmediate(10*time.Second, 10*time.Minute, func() (bool, error) {
|
||||
co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(context.Background(), "cluster", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
|
|
@ -143,7 +141,7 @@ var _ = Describe("ARO Operator - Internet checking", func() {
|
|||
})
|
||||
|
||||
var _ = Describe("ARO Operator - Geneva Logging", func() {
|
||||
Specify("genevalogging must be repaired if deployment deleted", func() {
|
||||
It("must be repaired if DaemonSet deleted", func() {
|
||||
mdsdReady := func() (bool, error) {
|
||||
done, err := ready.CheckDaemonSetIsReady(context.Background(), clients.Kubernetes.AppsV1().DaemonSets("openshift-azure-logging"), "mdsd")()
|
||||
if err != nil {
|
||||
|
|
@ -152,6 +150,7 @@ var _ = Describe("ARO Operator - Geneva Logging", func() {
|
|||
return done, nil // swallow error
|
||||
}
|
||||
|
||||
By("checking that mdsd DaemonSet is ready before the test")
|
||||
err := wait.PollImmediate(30*time.Second, 15*time.Minute, mdsdReady)
|
||||
if err != nil {
|
||||
// TODO: Remove dump once reason for flakes is clear
|
||||
|
|
@ -163,11 +162,11 @@ var _ = Describe("ARO Operator - Geneva Logging", func() {
|
|||
initial, err := updatedObjects(context.Background(), "openshift-azure-logging")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// delete the mdsd daemonset
|
||||
By("deleting mdsd DaemonSet")
|
||||
err = clients.Kubernetes.AppsV1().DaemonSets("openshift-azure-logging").Delete(context.Background(), "mdsd", metav1.DeleteOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// wait for it to be fixed
|
||||
By("checking that mdsd DaemonSet is ready")
|
||||
err = wait.PollImmediate(30*time.Second, 15*time.Minute, mdsdReady)
|
||||
if err != nil {
|
||||
// TODO: Remove dump once reason for flakes is clear
|
||||
|
|
@ -176,7 +175,7 @@ var _ = Describe("ARO Operator - Geneva Logging", func() {
|
|||
}
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// confirm that only one object was updated
|
||||
By("confirming that only one object was updated")
|
||||
final, err := updatedObjects(context.Background(), "openshift-azure-logging")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
if len(final)-len(initial) != 1 {
|
||||
|
|
@ -188,7 +187,7 @@ var _ = Describe("ARO Operator - Geneva Logging", func() {
|
|||
})
|
||||
|
||||
var _ = Describe("ARO Operator - Cluster Monitoring ConfigMap", func() {
|
||||
Specify("cluster monitoring configmap should not have persistent volume config", func() {
|
||||
It("must not have persistent volume set", func() {
|
||||
var cm *corev1.ConfigMap
|
||||
var err error
|
||||
configMapExists := func() (bool, error) {
|
||||
|
|
@ -199,9 +198,11 @@ var _ = Describe("ARO Operator - Cluster Monitoring ConfigMap", func() {
|
|||
return true, nil
|
||||
}
|
||||
|
||||
By("waiting for the ConfigMap to make sure it exists")
|
||||
err = wait.PollImmediate(30*time.Second, 15*time.Minute, configMapExists)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("unmarshalling the config from the ConfigMap data")
|
||||
var configData monitoring.Config
|
||||
configDataJSON, err := yaml.YAMLToJSON([]byte(cm.Data["config.yaml"]))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
|
@ -211,13 +212,14 @@ var _ = Describe("ARO Operator - Cluster Monitoring ConfigMap", func() {
|
|||
log.Warn(err)
|
||||
}
|
||||
|
||||
By("checking config correctness")
|
||||
Expect(configData.PrometheusK8s.Retention).To(BeEmpty())
|
||||
Expect(configData.PrometheusK8s.VolumeClaimTemplate).To(BeNil())
|
||||
Expect(configData.AlertManagerMain.VolumeClaimTemplate).To(BeNil())
|
||||
|
||||
})
|
||||
|
||||
Specify("cluster monitoring configmap should be restored if deleted", func() {
|
||||
It("must be restored if deleted", func() {
|
||||
configMapExists := func() (bool, error) {
|
||||
_, err := clients.Kubernetes.CoreV1().ConfigMaps("openshift-monitoring").Get(context.Background(), "cluster-monitoring-config", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
|
|
@ -226,22 +228,22 @@ var _ = Describe("ARO Operator - Cluster Monitoring ConfigMap", func() {
|
|||
return true, nil
|
||||
}
|
||||
|
||||
By("waiting for the ConfigMap to make sure it exists")
|
||||
err := wait.PollImmediate(30*time.Second, 15*time.Minute, configMapExists)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("deleting for the ConfigMap")
|
||||
err = clients.Kubernetes.CoreV1().ConfigMaps("openshift-monitoring").Delete(context.Background(), "cluster-monitoring-config", metav1.DeleteOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("waiting for the ConfigMap to make sure it was restored")
|
||||
err = wait.PollImmediate(30*time.Second, 15*time.Minute, configMapExists)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
_, err = clients.Kubernetes.CoreV1().ConfigMaps("openshift-monitoring").Get(context.Background(), "cluster-monitoring-config", metav1.GetOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("ARO Operator - RBAC", func() {
|
||||
Specify("system:aro-sre ClusterRole should be restored if deleted", func() {
|
||||
It("must restore system:aro-sre ClusterRole if deleted", func() {
|
||||
clusterRoleExists := func() (bool, error) {
|
||||
_, err := clients.Kubernetes.RbacV1().ClusterRoles().Get(context.Background(), "system:aro-sre", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
|
|
@ -250,22 +252,22 @@ var _ = Describe("ARO Operator - RBAC", func() {
|
|||
return true, nil
|
||||
}
|
||||
|
||||
By("waiting for the ClusterRole to make sure it exists")
|
||||
err := wait.PollImmediate(30*time.Second, 15*time.Minute, clusterRoleExists)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("deleting for the ClusterRole")
|
||||
err = clients.Kubernetes.RbacV1().ClusterRoles().Delete(context.Background(), "system:aro-sre", metav1.DeleteOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("waiting for the ClusterRole to make sure it was restored")
|
||||
err = wait.PollImmediate(30*time.Second, 15*time.Minute, clusterRoleExists)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
_, err = clients.Kubernetes.RbacV1().ClusterRoles().Get(context.Background(), "system:aro-sre", metav1.GetOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("ARO Operator - Conditions", func() {
|
||||
Specify("Cluster check conditions should not be failing", func() {
|
||||
It("must have all the conditions set to true", func() {
|
||||
// Save the last got conditions so that we can print them in the case of
|
||||
// the test failing
|
||||
var lastConditions []operatorv1.OperatorCondition
|
||||
|
|
@ -297,8 +299,8 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() {
|
|||
|
||||
const nsg = "e2e-nsg"
|
||||
|
||||
// Gathers vnet name, resource group, location, and adds master/worker subnets to list to reconcile.
|
||||
gatherNetworkInfo := func() {
|
||||
By("gathering vnet name, resource group, location, and adds master/worker subnets to list to reconcile")
|
||||
oc, err := clients.OpenshiftClustersv20200430.Get(ctx, vnetResourceGroup, clusterName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
location = *oc.Location
|
||||
|
|
@ -319,8 +321,8 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() {
|
|||
vnetName = r.ResourceName
|
||||
}
|
||||
|
||||
// Creates an empty NSG that gets assigned to master/worker subnets.
|
||||
createE2ENSG := func() {
|
||||
By("creating an empty test NSG")
|
||||
testnsg = mgmtnetwork.SecurityGroup{
|
||||
Location: &location,
|
||||
Name: to.StringPtr(nsg),
|
||||
|
|
@ -329,6 +331,8 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() {
|
|||
}
|
||||
err := clients.NetworkSecurityGroups.CreateOrUpdateAndWait(ctx, resourceGroup, nsg, testnsg)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("getting the freshly created test NSG resource")
|
||||
testnsg, err = clients.NetworkSecurityGroups.Get(ctx, resourceGroup, nsg, "")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
|
@ -338,6 +342,7 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() {
|
|||
createE2ENSG()
|
||||
})
|
||||
AfterEach(func() {
|
||||
By("deleting test NSG")
|
||||
err := clients.NetworkSecurityGroups.DeleteAndWait(context.Background(), resourceGroup, nsg)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
|
|
@ -345,6 +350,7 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() {
|
|||
})
|
||||
It("must reconcile list of subnets when NSG is changed", func() {
|
||||
for subnet := range subnetsToReconcile {
|
||||
By(fmt.Sprintf("assigning test NSG to subnet %q", subnet))
|
||||
// Gets current subnet NSG and then updates it to testnsg.
|
||||
subnetObject, err := clients.Subnet.Get(ctx, resourceGroup, vnetName, subnet, "")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
|
@ -354,18 +360,17 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() {
|
|||
err = clients.Subnet.CreateOrUpdateAndWait(ctx, resourceGroup, vnetName, subnet, subnetObject)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
for subnet, correctNSG := range subnetsToReconcile {
|
||||
// Validate subnet reconciles to original NSG.
|
||||
By(fmt.Sprintf("waiting for the subnet %q to be reconciled so it includes the original cluster NSG", subnet))
|
||||
err := wait.PollImmediate(30*time.Second, 10*time.Minute, func() (bool, error) {
|
||||
s, err := clients.Subnet.Get(ctx, resourceGroup, vnetName, subnet, "")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if *s.NetworkSecurityGroup.ID == *correctNSG {
|
||||
log.Infof("%s subnet's nsg matched expected value", subnet)
|
||||
return true, nil
|
||||
}
|
||||
log.Errorf("%s nsg: %s did not match expected value: %s", subnet, *s.NetworkSecurityGroup.ID, *correctNSG)
|
||||
return false, nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
|
@ -376,8 +381,9 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() {
|
|||
var _ = Describe("ARO Operator - MUO Deployment", func() {
|
||||
ctx := context.Background()
|
||||
|
||||
Specify("MUO should be deployed by default", func() {
|
||||
It("must be deployed by default with FIPS crypto mandated", func() {
|
||||
muoIsDeployed := func() (bool, error) {
|
||||
By("getting MUO pods")
|
||||
pods, err := clients.Kubernetes.CoreV1().Pods("openshift-managed-upgrade-operator").List(ctx, metav1.ListOptions{
|
||||
LabelSelector: "name=managed-upgrade-operator",
|
||||
})
|
||||
|
|
@ -387,12 +393,14 @@ var _ = Describe("ARO Operator - MUO Deployment", func() {
|
|||
if len(pods.Items) != 1 {
|
||||
return false, fmt.Errorf("%d managed-upgrade-operator pods found", len(pods.Items))
|
||||
}
|
||||
|
||||
By("getting logs from MUO")
|
||||
b, err := clients.Kubernetes.CoreV1().Pods("openshift-managed-upgrade-operator").GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{}).DoRaw(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// check for mandated FIPS
|
||||
By("verifying that MUO has FIPS crypto mandated by reading logs")
|
||||
return strings.Contains(string(b), `msg="FIPS crypto mandated: true"`), nil
|
||||
}
|
||||
|
||||
|
|
@ -401,28 +409,6 @@ var _ = Describe("ARO Operator - MUO Deployment", func() {
|
|||
})
|
||||
})
|
||||
|
||||
var _ = Describe("ARO Operator - MHC Deployment", func() {
|
||||
Specify("MHC should be enabled and managed by default", func() {
|
||||
mhcIsDeployed := func() (bool, error) {
|
||||
co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(context.Background(), "cluster", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
mhcEnabled, _ := strconv.ParseBool(co.Spec.OperatorFlags.GetWithDefault("aro.machinehealthcheck.enabled", "false"))
|
||||
mhcManaged, _ := strconv.ParseBool(co.Spec.OperatorFlags.GetWithDefault("aro.machinehealthcheck.managed", "false"))
|
||||
|
||||
if mhcEnabled && mhcManaged {
|
||||
return true, nil
|
||||
}
|
||||
return false, errors.New("mhc should be enabled and managed by default")
|
||||
}
|
||||
|
||||
err := wait.PollImmediate(30*time.Second, 10*time.Minute, mhcIsDeployed)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("ARO Operator - ImageConfig Reconciler", func() {
|
||||
const (
|
||||
imageconfigFlag = "aro.imageconfig.enabled"
|
||||
|
|
@ -434,22 +420,6 @@ var _ = Describe("ARO Operator - ImageConfig Reconciler", func() {
|
|||
var requiredRegistries []string
|
||||
var imageconfig *configv1.Image
|
||||
|
||||
// Reimplementation of function from image config controller
|
||||
getCloudAwareRegistries := func(instance *arov1alpha1.Cluster) ([]string, error) {
|
||||
var requiredRegistries []string
|
||||
switch instance.Spec.AZEnvironment {
|
||||
case azureclient.PublicCloud.Environment.Name:
|
||||
requiredRegistries = []string{instance.Spec.ACRDomain, "arosvc." + instance.Spec.Location + ".data." + azure.PublicCloud.ContainerRegistryDNSSuffix}
|
||||
|
||||
case azureclient.USGovernmentCloud.Environment.Name:
|
||||
requiredRegistries = []string{instance.Spec.ACRDomain, "arosvc." + instance.Spec.Location + ".data." + azure.USGovernmentCloud.ContainerRegistryDNSSuffix}
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("cloud environment %s is not supported", instance.Spec.AZEnvironment)
|
||||
}
|
||||
return requiredRegistries, nil
|
||||
}
|
||||
|
||||
sliceEqual := func(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
|
|
@ -466,6 +436,7 @@ var _ = Describe("ARO Operator - ImageConfig Reconciler", func() {
|
|||
}
|
||||
|
||||
verifyLists := func(expectedAllowlist, expectedBlocklist []string) (bool, error) {
|
||||
By("getting the actual Image config state")
|
||||
// have to do this because using declaration assignment in following line results in pre-declared imageconfig var not being used
|
||||
var err error
|
||||
imageconfig, err = clients.ConfigClient.ConfigV1().Images().Get(ctx, "cluster", metav1.GetOptions{})
|
||||
|
|
@ -474,10 +445,12 @@ var _ = Describe("ARO Operator - ImageConfig Reconciler", func() {
|
|||
allowList := imageconfig.Spec.RegistrySources.AllowedRegistries
|
||||
blockList := imageconfig.Spec.RegistrySources.BlockedRegistries
|
||||
|
||||
By("comparing the actual allow and block lists with expected lists")
|
||||
return sliceEqual(allowList, expectedAllowlist) && sliceEqual(blockList, expectedBlocklist), nil
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
By("checking whether Image config reconciliation is enabled in ARO operator config")
|
||||
instance, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
|
|
@ -485,55 +458,59 @@ var _ = Describe("ARO Operator - ImageConfig Reconciler", func() {
|
|||
Skip("ImageConfig Controller is not enabled, skipping test")
|
||||
}
|
||||
|
||||
imageconfig, err = clients.ConfigClient.ConfigV1().Images().Get(ctx, "cluster", metav1.GetOptions{})
|
||||
By("getting a list of required registries from the ARO operator config")
|
||||
requiredRegistries, err = imageController.GetCloudAwareRegistries(instance)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
requiredRegistries, err = getCloudAwareRegistries(instance)
|
||||
By("getting the Image config")
|
||||
imageconfig, err = clients.ConfigClient.ConfigV1().Images().Get(ctx, "cluster", metav1.GetOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
By("resetting Image config")
|
||||
imageconfig.Spec.RegistrySources.AllowedRegistries = nil
|
||||
imageconfig.Spec.RegistrySources.BlockedRegistries = nil
|
||||
|
||||
_, err := clients.ConfigClient.ConfigV1().Images().Update(ctx, imageconfig, metav1.UpdateOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("waiting for the Image config to be reset")
|
||||
Eventually(func(g Gomega) {
|
||||
g.Expect(verifyLists(nil, nil)).To(BeTrue())
|
||||
}).WithTimeout(timeout).Should(Succeed())
|
||||
})
|
||||
|
||||
It("should have empty allow and block lists by default", func() {
|
||||
It("must set empty allow and block lists in Image config by default", func() {
|
||||
allowList := imageconfig.Spec.RegistrySources.AllowedRegistries
|
||||
blockList := imageconfig.Spec.RegistrySources.BlockedRegistries
|
||||
|
||||
By("checking that the allow and block lists are empty")
|
||||
Expect(allowList).To(BeEmpty())
|
||||
Expect(blockList).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("should add registries to the allow list", func() {
|
||||
|
||||
expectedAllowlist := append(requiredRegistries, optionalRegistry)
|
||||
|
||||
It("must add the ARO service registries to the allow list alongside the customer added registries", func() {
|
||||
By("adding the test registry to the allow list of the Image config")
|
||||
imageconfig.Spec.RegistrySources.AllowedRegistries = append(imageconfig.Spec.RegistrySources.AllowedRegistries, optionalRegistry)
|
||||
|
||||
_, err := clients.ConfigClient.ConfigV1().Images().Update(ctx, imageconfig, metav1.UpdateOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("checking that Image config eventually has ARO service registries and the test registry in the allow list")
|
||||
expectedAllowlist := append(requiredRegistries, optionalRegistry)
|
||||
Eventually(func(g Gomega) {
|
||||
g.Expect(verifyLists(expectedAllowlist, nil)).To(BeTrue())
|
||||
}).WithTimeout(timeout).Should(Succeed())
|
||||
})
|
||||
|
||||
It("should add optional registries to the block list and skip required registries", func() {
|
||||
expectedBlocklist := []string{optionalRegistry}
|
||||
|
||||
It("must remove ARO service registries from the block lists, but keep customer added registries", func() {
|
||||
By("adding the test registry and one of the ARO service registry to the block list of the Image config")
|
||||
imageconfig.Spec.RegistrySources.BlockedRegistries = append(imageconfig.Spec.RegistrySources.BlockedRegistries, optionalRegistry, requiredRegistries[0])
|
||||
|
||||
_, err := clients.ConfigClient.ConfigV1().Images().Update(ctx, imageconfig, metav1.UpdateOptions{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("checking that Image config eventually doesn't include ARO service registries")
|
||||
expectedBlocklist := []string{optionalRegistry}
|
||||
Eventually(func(g Gomega) {
|
||||
g.Expect(verifyLists(nil, expectedBlocklist)).To(BeTrue())
|
||||
}).WithTimeout(timeout).Should(Succeed())
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo" //nolint
|
||||
. "github.com/onsi/ginkgo/v2" //nolint
|
||||
. "github.com/onsi/gomega" //nolint
|
||||
. "github.com/tebeka/selenium" //nolint
|
||||
|
||||
|
|
@ -39,23 +39,16 @@ import (
|
|||
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features"
|
||||
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network"
|
||||
redhatopenshift20200430 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2020-04-30/redhatopenshift"
|
||||
redhatopenshift20210901preview "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2021-09-01-preview/redhatopenshift"
|
||||
redhatopenshift20220401 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2022-04-01/redhatopenshift"
|
||||
redhatopenshift20220904 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2022-09-04/redhatopenshift"
|
||||
"github.com/Azure/ARO-RP/pkg/util/cluster"
|
||||
utillog "github.com/Azure/ARO-RP/pkg/util/log"
|
||||
"github.com/Azure/ARO-RP/test/util/kubeadminkubeconfig"
|
||||
)
|
||||
|
||||
type clientSet struct {
|
||||
OpenshiftClustersv20200430 redhatopenshift20200430.OpenShiftClustersClient
|
||||
Operationsv20200430 redhatopenshift20200430.OperationsClient
|
||||
OpenshiftClustersv20210901preview redhatopenshift20210901preview.OpenShiftClustersClient
|
||||
Operationsv20210901preview redhatopenshift20210901preview.OperationsClient
|
||||
OpenshiftClustersv20220401 redhatopenshift20220401.OpenShiftClustersClient
|
||||
Operationsv20220401 redhatopenshift20220401.OperationsClient
|
||||
OpenshiftClustersv20220904 redhatopenshift20220904.OpenShiftClustersClient
|
||||
Operationsv20220904 redhatopenshift20220904.OperationsClient
|
||||
OpenshiftClustersv20200430 redhatopenshift20200430.OpenShiftClustersClient
|
||||
Operationsv20200430 redhatopenshift20200430.OperationsClient
|
||||
OpenshiftClustersv20220401 redhatopenshift20220401.OpenShiftClustersClient
|
||||
|
||||
VirtualMachines compute.VirtualMachinesClient
|
||||
Resources features.ResourcesClient
|
||||
|
|
@ -294,12 +287,9 @@ func newClientSet(ctx context.Context) (*clientSet, error) {
|
|||
}
|
||||
|
||||
return &clientSet{
|
||||
OpenshiftClustersv20200430: redhatopenshift20200430.NewOpenShiftClustersClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
Operationsv20200430: redhatopenshift20200430.NewOperationsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
OpenshiftClustersv20210901preview: redhatopenshift20210901preview.NewOpenShiftClustersClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
Operationsv20210901preview: redhatopenshift20210901preview.NewOperationsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
OpenshiftClustersv20220401: redhatopenshift20220401.NewOpenShiftClustersClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
Operationsv20220401: redhatopenshift20220401.NewOperationsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
OpenshiftClustersv20200430: redhatopenshift20200430.NewOpenShiftClustersClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
Operationsv20200430: redhatopenshift20200430.NewOperationsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
OpenshiftClustersv20220401: redhatopenshift20220401.NewOpenShiftClustersClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
|
||||
VirtualMachines: compute.NewVirtualMachinesClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
Resources: features.NewResourcesClient(_env.Environment(), _env.SubscriptionID(), authorizer),
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ github.com/Azure/azure-sdk-for-go/profiles/2018-03-01/resources/mgmt/subscriptio
|
|||
github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-03-30/compute
|
||||
github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute
|
||||
github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute
|
||||
github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2021-10-01/containerservice
|
||||
github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2021-01-15/documentdb
|
||||
github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2016-04-01/dns
|
||||
github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2018-05-01/dns
|
||||
|
|
@ -493,6 +494,9 @@ github.com/go-playground/validator/v10
|
|||
# github.com/go-stack/stack v1.8.1
|
||||
## explicit; go 1.17
|
||||
github.com/go-stack/stack
|
||||
# github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0
|
||||
## explicit; go 1.13
|
||||
github.com/go-task/slim-sprig
|
||||
# github.com/go-test/deep v1.0.8
|
||||
## explicit; go 1.16
|
||||
github.com/go-test/deep
|
||||
|
|
@ -637,6 +641,9 @@ github.com/google/go-cmp/cmp/internal/value
|
|||
## explicit; go 1.12
|
||||
github.com/google/gofuzz
|
||||
github.com/google/gofuzz/bytesource
|
||||
# github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1
|
||||
## explicit; go 1.14
|
||||
github.com/google/pprof/profile
|
||||
# github.com/google/renameio v1.0.1
|
||||
## explicit; go 1.13
|
||||
github.com/google/renameio
|
||||
|
|
@ -946,41 +953,34 @@ github.com/nishanths/exhaustive
|
|||
# github.com/nishanths/predeclared v0.2.1
|
||||
## explicit; go 1.14
|
||||
github.com/nishanths/predeclared/passes/predeclared
|
||||
# github.com/nxadm/tail v1.4.8
|
||||
## explicit; go 1.13
|
||||
github.com/nxadm/tail
|
||||
github.com/nxadm/tail/ratelimiter
|
||||
github.com/nxadm/tail/util
|
||||
github.com/nxadm/tail/watch
|
||||
github.com/nxadm/tail/winfile
|
||||
# github.com/oklog/ulid v1.3.1
|
||||
## explicit
|
||||
github.com/oklog/ulid
|
||||
# github.com/olekukonko/tablewriter v0.0.5
|
||||
## explicit; go 1.12
|
||||
github.com/olekukonko/tablewriter
|
||||
# github.com/onsi/ginkgo v1.16.5
|
||||
# github.com/onsi/ginkgo/v2 v2.1.3
|
||||
## explicit; go 1.16
|
||||
github.com/onsi/ginkgo
|
||||
github.com/onsi/ginkgo/config
|
||||
github.com/onsi/ginkgo/formatter
|
||||
github.com/onsi/ginkgo/internal/codelocation
|
||||
github.com/onsi/ginkgo/internal/containernode
|
||||
github.com/onsi/ginkgo/internal/failer
|
||||
github.com/onsi/ginkgo/internal/global
|
||||
github.com/onsi/ginkgo/internal/leafnodes
|
||||
github.com/onsi/ginkgo/internal/remote
|
||||
github.com/onsi/ginkgo/internal/spec
|
||||
github.com/onsi/ginkgo/internal/spec_iterator
|
||||
github.com/onsi/ginkgo/internal/specrunner
|
||||
github.com/onsi/ginkgo/internal/suite
|
||||
github.com/onsi/ginkgo/internal/testingtproxy
|
||||
github.com/onsi/ginkgo/internal/writer
|
||||
github.com/onsi/ginkgo/reporters
|
||||
github.com/onsi/ginkgo/reporters/stenographer
|
||||
github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable
|
||||
github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty
|
||||
github.com/onsi/ginkgo/types
|
||||
github.com/onsi/ginkgo/v2
|
||||
github.com/onsi/ginkgo/v2/config
|
||||
github.com/onsi/ginkgo/v2/formatter
|
||||
github.com/onsi/ginkgo/v2/ginkgo
|
||||
github.com/onsi/ginkgo/v2/ginkgo/build
|
||||
github.com/onsi/ginkgo/v2/ginkgo/command
|
||||
github.com/onsi/ginkgo/v2/ginkgo/generators
|
||||
github.com/onsi/ginkgo/v2/ginkgo/internal
|
||||
github.com/onsi/ginkgo/v2/ginkgo/labels
|
||||
github.com/onsi/ginkgo/v2/ginkgo/outline
|
||||
github.com/onsi/ginkgo/v2/ginkgo/run
|
||||
github.com/onsi/ginkgo/v2/ginkgo/unfocus
|
||||
github.com/onsi/ginkgo/v2/ginkgo/watch
|
||||
github.com/onsi/ginkgo/v2/internal
|
||||
github.com/onsi/ginkgo/v2/internal/global
|
||||
github.com/onsi/ginkgo/v2/internal/interrupt_handler
|
||||
github.com/onsi/ginkgo/v2/internal/parallel_support
|
||||
github.com/onsi/ginkgo/v2/internal/testingtproxy
|
||||
github.com/onsi/ginkgo/v2/reporters
|
||||
github.com/onsi/ginkgo/v2/types
|
||||
# github.com/onsi/gomega v1.19.0
|
||||
## explicit; go 1.18
|
||||
github.com/onsi/gomega
|
||||
|
|
@ -1093,9 +1093,12 @@ github.com/openshift/console-operator/pkg/api
|
|||
# github.com/openshift/hive v1.1.16 => github.com/openshift/hive v1.1.17-0.20220719141355-c63c9b0281d8
|
||||
## explicit; go 1.18
|
||||
github.com/openshift/hive/pkg/client/clientset/versioned
|
||||
github.com/openshift/hive/pkg/client/clientset/versioned/fake
|
||||
github.com/openshift/hive/pkg/client/clientset/versioned/scheme
|
||||
github.com/openshift/hive/pkg/client/clientset/versioned/typed/hive/v1
|
||||
github.com/openshift/hive/pkg/client/clientset/versioned/typed/hive/v1/fake
|
||||
github.com/openshift/hive/pkg/client/clientset/versioned/typed/hiveinternal/v1alpha1
|
||||
github.com/openshift/hive/pkg/client/clientset/versioned/typed/hiveinternal/v1alpha1/fake
|
||||
# github.com/openshift/hive/apis v0.0.0 => github.com/openshift/hive/apis v0.0.0-20220719141355-c63c9b0281d8
|
||||
## explicit; go 1.18
|
||||
github.com/openshift/hive/apis/hive/v1
|
||||
|
|
@ -1589,6 +1592,7 @@ golang.org/x/sys/windows
|
|||
golang.org/x/term
|
||||
# golang.org/x/text v0.3.7
|
||||
## explicit; go 1.17
|
||||
golang.org/x/text/cases
|
||||
golang.org/x/text/encoding
|
||||
golang.org/x/text/encoding/charmap
|
||||
golang.org/x/text/encoding/htmlindex
|
||||
|
|
@ -1599,6 +1603,7 @@ golang.org/x/text/encoding/korean
|
|||
golang.org/x/text/encoding/simplifiedchinese
|
||||
golang.org/x/text/encoding/traditionalchinese
|
||||
golang.org/x/text/encoding/unicode
|
||||
golang.org/x/text/internal
|
||||
golang.org/x/text/internal/language
|
||||
golang.org/x/text/internal/language/compact
|
||||
golang.org/x/text/internal/tag
|
||||
|
|
@ -1823,9 +1828,6 @@ gopkg.in/ini.v1
|
|||
gopkg.in/square/go-jose.v2
|
||||
gopkg.in/square/go-jose.v2/cipher
|
||||
gopkg.in/square/go-jose.v2/json
|
||||
# gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
|
||||
## explicit
|
||||
gopkg.in/tomb.v1
|
||||
# gopkg.in/yaml.v2 v2.4.0
|
||||
## explicit; go 1.15
|
||||
gopkg.in/yaml.v2
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче