Update container resolv.conf when host network changes /etc/resolv.conf

Only modifies non-running containers resolv.conf bind mount, and only if
the container has an unmodified resolv.conf compared to its contents at
container start time (so we don't overwrite manual/automated changes
within the container runtime). For containers which are running when
the host resolv.conf changes, the update will only be applied to the
container version of resolv.conf when the container is "bounced" down
and back up (e.g. stop/start or restart)

Docker-DCO-1.1-Signed-off-by: Phil Estes <estesp@linux.vnet.ibm.com> (github: estesp)
This commit is contained in:
Phil Estes 2014-12-10 00:55:09 -05:00
Родитель edbf1ed680
Коммит 63a7ccdd23
8 изменённых файлов: 426 добавлений и 51 удалений

Просмотреть файл

@ -81,6 +81,7 @@ type Container struct {
MountLabel, ProcessLabel string
AppArmorProfile string
RestartCount int
UpdateDns bool
// Maps container paths to volume paths. The key in this is the path to which
// the volume is being mounted inside the container. Value is the path of the
@ -945,6 +946,29 @@ func (container *Container) DisableLink(name string) {
func (container *Container) setupContainerDns() error {
if container.ResolvConfPath != "" {
// check if this is an existing container that needs DNS update:
if container.UpdateDns {
// read the host's resolv.conf, get the hash and call updateResolvConf
log.Debugf("Check container (%s) for update to resolv.conf - UpdateDns flag was set", container.ID)
latestResolvConf, latestHash := resolvconf.GetLastModified()
// because the new host resolv.conf might have localhost nameservers..
updatedResolvConf, modified := resolvconf.RemoveReplaceLocalDns(latestResolvConf)
if modified {
// changes have occurred during resolv.conf localhost cleanup: generate an updated hash
newHash, err := utils.HashData(bytes.NewReader(updatedResolvConf))
if err != nil {
return err
}
latestHash = newHash
}
if err := container.updateResolvConf(updatedResolvConf, latestHash); err != nil {
return err
}
// successful update of the restarting container; set the flag off
container.UpdateDns = false
}
return nil
}
@ -983,17 +1007,86 @@ func (container *Container) setupContainerDns() error {
}
// replace any localhost/127.* nameservers
resolvConf = utils.RemoveLocalDns(resolvConf)
// if the resulting resolvConf is empty, use DefaultDns
if !bytes.Contains(resolvConf, []byte("nameserver")) {
log.Infof("No non localhost DNS resolver found in resolv.conf and containers can't use it. Using default external servers : %v", DefaultDns)
// prefix the default dns options with nameserver
resolvConf = append(resolvConf, []byte("\nnameserver "+strings.Join(DefaultDns, "\nnameserver "))...)
}
resolvConf, _ = resolvconf.RemoveReplaceLocalDns(resolvConf)
}
//get a sha256 hash of the resolv conf at this point so we can check
//for changes when the host resolv.conf changes (e.g. network update)
resolvHash, err := utils.HashData(bytes.NewReader(resolvConf))
if err != nil {
return err
}
resolvHashFile := container.ResolvConfPath + ".hash"
if err = ioutil.WriteFile(resolvHashFile, []byte(resolvHash), 0644); err != nil {
return err
}
return ioutil.WriteFile(container.ResolvConfPath, resolvConf, 0644)
}
// called when the host's resolv.conf changes to check whether container's resolv.conf
// is unchanged by the container "user" since container start: if unchanged, the
// container's resolv.conf will be updated to match the host's new resolv.conf
func (container *Container) updateResolvConf(updatedResolvConf []byte, newResolvHash string) error {
if container.ResolvConfPath == "" {
return nil
}
if container.Running {
//set a marker in the hostConfig to update on next start/restart
container.UpdateDns = true
return nil
}
resolvHashFile := container.ResolvConfPath + ".hash"
//read the container's current resolv.conf and compute the hash
resolvBytes, err := ioutil.ReadFile(container.ResolvConfPath)
if err != nil {
return err
}
curHash, err := utils.HashData(bytes.NewReader(resolvBytes))
if err != nil {
return err
}
//read the hash from the last time we wrote resolv.conf in the container
hashBytes, err := ioutil.ReadFile(resolvHashFile)
if err != nil {
return err
}
//if the user has not modified the resolv.conf of the container since we wrote it last
//we will replace it with the updated resolv.conf from the host
if string(hashBytes) == curHash {
log.Debugf("replacing %q with updated host resolv.conf", container.ResolvConfPath)
// for atomic updates to these files, use temporary files with os.Rename:
dir := path.Dir(container.ResolvConfPath)
tmpHashFile, err := ioutil.TempFile(dir, "hash")
if err != nil {
return err
}
tmpResolvFile, err := ioutil.TempFile(dir, "resolv")
if err != nil {
return err
}
// write the updates to the temp files
if err = ioutil.WriteFile(tmpHashFile.Name(), []byte(newResolvHash), 0644); err != nil {
return err
}
if err = ioutil.WriteFile(tmpResolvFile.Name(), updatedResolvConf, 0644); err != nil {
return err
}
// rename the temp files for atomic replace
if err = os.Rename(tmpHashFile.Name(), resolvHashFile); err != nil {
return err
}
return os.Rename(tmpResolvFile.Name(), container.ResolvConfPath)
}
return nil
}
func (container *Container) updateParentsHosts() error {
parents, err := container.daemon.Parents(container.Name)
if err != nil {

Просмотреть файл

@ -1,6 +1,7 @@
package daemon
import (
"bytes"
"fmt"
"io"
"io/ioutil"
@ -32,6 +33,7 @@ import (
"github.com/docker/docker/pkg/graphdb"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/namesgenerator"
"github.com/docker/docker/pkg/networkfs/resolvconf"
"github.com/docker/docker/pkg/parsers"
"github.com/docker/docker/pkg/parsers/kernel"
"github.com/docker/docker/pkg/sysinfo"
@ -40,10 +42,11 @@ import (
"github.com/docker/docker/trust"
"github.com/docker/docker/utils"
"github.com/docker/docker/volumes"
"github.com/go-fsnotify/fsnotify"
)
var (
DefaultDns = []string{"8.8.8.8", "8.8.4.4"}
validContainerNameChars = `[a-zA-Z0-9][a-zA-Z0-9_.-]`
validContainerNamePattern = regexp.MustCompile(`^/?` + validContainerNameChars + `+$`)
)
@ -402,6 +405,60 @@ func (daemon *Daemon) restore() error {
return nil
}
// set up the watch on the host's /etc/resolv.conf so that we can update container's
// live resolv.conf when the network changes on the host
func (daemon *Daemon) setupResolvconfWatcher() error {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return err
}
//this goroutine listens for the events on the watch we add
//on the resolv.conf file on the host
go func() {
for {
select {
case event := <-watcher.Events:
if event.Op&fsnotify.Write == fsnotify.Write {
// verify a real change happened before we go further--a file write may have happened
// without an actual change to the file
updatedResolvConf, newResolvConfHash, err := resolvconf.GetIfChanged()
if err != nil {
log.Debugf("Error retrieving updated host resolv.conf: %v", err)
} else if updatedResolvConf != nil {
// because the new host resolv.conf might have localhost nameservers..
updatedResolvConf, modified := resolvconf.RemoveReplaceLocalDns(updatedResolvConf)
if modified {
// changes have occurred during localhost cleanup: generate an updated hash
newHash, err := utils.HashData(bytes.NewReader(updatedResolvConf))
if err != nil {
log.Debugf("Error generating hash of new resolv.conf: %v", err)
} else {
newResolvConfHash = newHash
}
}
log.Debugf("host network resolv.conf changed--walking container list for updates")
contList := daemon.containers.List()
for _, container := range contList {
if err := container.updateResolvConf(updatedResolvConf, newResolvConfHash); err != nil {
log.Debugf("Error on resolv.conf update check for container ID: %s: %v", container.ID, err)
}
}
}
}
case err := <-watcher.Errors:
log.Debugf("host resolv.conf notify error: %v", err)
}
}
}()
if err := watcher.Add("/etc/resolv.conf"); err != nil {
return err
}
return nil
}
func (daemon *Daemon) checkDeprecatedExpose(config *runconfig.Config) bool {
if config != nil {
if config.PortSpecs != nil {
@ -924,6 +981,12 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error)
if err := daemon.restore(); err != nil {
return nil, err
}
// set up filesystem watch on resolv.conf for network changes
if err := daemon.setupResolvconfWatcher(); err != nil {
return nil, err
}
// Setup shutdown handlers
// FIXME: can these shutdown handlers be registered closer to their source?
eng.OnShutdown(func() {

Просмотреть файл

@ -24,34 +24,3 @@ func TestMergeLxcConfig(t *testing.T) {
t.Fatalf("expected %s got %s", expected, cpuset)
}
}
func TestRemoveLocalDns(t *testing.T) {
ns0 := "nameserver 10.16.60.14\nnameserver 10.16.60.21\n"
if result := utils.RemoveLocalDns([]byte(ns0)); result != nil {
if ns0 != string(result) {
t.Fatalf("Failed No Localhost: expected \n<%s> got \n<%s>", ns0, string(result))
}
}
ns1 := "nameserver 10.16.60.14\nnameserver 10.16.60.21\nnameserver 127.0.0.1\n"
if result := utils.RemoveLocalDns([]byte(ns1)); result != nil {
if ns0 != string(result) {
t.Fatalf("Failed Localhost: expected \n<%s> got \n<%s>", ns0, string(result))
}
}
ns1 = "nameserver 10.16.60.14\nnameserver 127.0.0.1\nnameserver 10.16.60.21\n"
if result := utils.RemoveLocalDns([]byte(ns1)); result != nil {
if ns0 != string(result) {
t.Fatalf("Failed Localhost: expected \n<%s> got \n<%s>", ns0, string(result))
}
}
ns1 = "nameserver 127.0.1.1\nnameserver 10.16.60.14\nnameserver 10.16.60.21\n"
if result := utils.RemoveLocalDns([]byte(ns1)); result != nil {
if ns0 != string(result) {
t.Fatalf("Failed Localhost: expected \n<%s> got \n<%s>", ns0, string(result))
}
}
}

Просмотреть файл

@ -130,7 +130,7 @@ information. You can see this by running `mount` inside a container:
...
/dev/disk/by-uuid/1fec...ebdf on /etc/hostname type ext4 ...
/dev/disk/by-uuid/1fec...ebdf on /etc/hosts type ext4 ...
tmpfs on /etc/resolv.conf type tmpfs ...
/dev/disk/by-uuid/1fec...ebdf on /etc/resolv.conf type ext4 ...
...
This arrangement allows Docker to do clever things like keep
@ -178,7 +178,20 @@ Four different options affect container domain name services.
Note that Docker, in the absence of either of the last two options
above, will make `/etc/resolv.conf` inside of each container look like
the `/etc/resolv.conf` of the host machine where the `docker` daemon is
running. The options then modify this default configuration.
running. You might wonder what happens when the host machine's
`/etc/resolv.conf` file changes. The `docker` daemon has a file change
notifier active which will watch for changes to the host DNS configuration.
When the host file changes, all stopped containers which have a matching
`resolv.conf` to the host will be updated immediately to this newest host
configuration. Containers which are running when the host configuration
changes will need to stop and start to pick up the host changes due to lack
of a facility to ensure atomic writes of the `resolv.conf` file while the
container is running. If the container's `resolv.conf` has been edited since
it was started with the default configuration, no replacement will be
attempted as it would overwrite the changes performed by the container.
If the options (`--dns` or `--dns-search`) have been used to modify the
default host configuration, then the replacement with an updated host's
`/etc/resolv.conf` will not happen as well.
## Communication between containers and the wider world

Просмотреть файл

@ -1403,6 +1403,157 @@ func TestRunDnsOptionsBasedOnHostResolvConf(t *testing.T) {
logDone("run - dns options based on host resolv.conf")
}
// Test the file watch notifier on docker host's /etc/resolv.conf
// A go-routine is responsible for auto-updating containers which are
// stopped and have an unmodified copy of resolv.conf, as well as
// marking running containers as requiring an update on next restart
func TestRunResolvconfUpdater(t *testing.T) {
tmpResolvConf := []byte("search pommesfrites.fr\nnameserver 12.34.56.78")
tmpLocalhostResolvConf := []byte("nameserver 127.0.0.1")
//take a copy of resolv.conf for restoring after test completes
resolvConfSystem, err := ioutil.ReadFile("/etc/resolv.conf")
if err != nil {
t.Fatal(err)
}
//cleanup
defer func() {
deleteAllContainers()
if err := ioutil.WriteFile("/etc/resolv.conf", resolvConfSystem, 0644); err != nil {
t.Fatal(err)
}
}()
//1. test that a non-running container gets an updated resolv.conf
cmd := exec.Command(dockerBinary, "run", "--name='first'", "busybox", "true")
if _, err := runCommand(cmd); err != nil {
t.Fatal(err)
}
containerID1, err := getIDByName("first")
if err != nil {
t.Fatal(err)
}
// replace resolv.conf with our temporary copy
bytesResolvConf := []byte(tmpResolvConf)
if err := ioutil.WriteFile("/etc/resolv.conf", bytesResolvConf, 0644); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second / 2)
// check for update in container
containerResolv, err := readContainerFile(containerID1, "resolv.conf")
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(containerResolv, bytesResolvConf) {
t.Fatalf("Stopped container does not have updated resolv.conf; expected %q, got %q", tmpResolvConf, string(containerResolv))
}
//2. test that a non-running container does not receive resolv.conf updates
// if it modified the container copy of the starting point resolv.conf
cmd = exec.Command(dockerBinary, "run", "--name='second'", "busybox", "sh", "-c", "echo 'search mylittlepony.com' >>/etc/resolv.conf")
if _, err = runCommand(cmd); err != nil {
t.Fatal(err)
}
containerID2, err := getIDByName("second")
if err != nil {
t.Fatal(err)
}
containerResolvHashBefore, err := readContainerFile(containerID2, "resolv.conf.hash")
if err != nil {
t.Fatal(err)
}
//make a change to resolv.conf (in this case replacing our tmp copy with orig copy)
if err := ioutil.WriteFile("/etc/resolv.conf", resolvConfSystem, 0644); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second / 2)
containerResolvHashAfter, err := readContainerFile(containerID2, "resolv.conf.hash")
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(containerResolvHashBefore, containerResolvHashAfter) {
t.Fatalf("Stopped container with modified resolv.conf should not have been updated; expected hash: %v, new hash: %v", containerResolvHashBefore, containerResolvHashAfter)
}
//3. test that a running container's resolv.conf is not modified while running
cmd = exec.Command(dockerBinary, "run", "-d", "busybox", "top")
out, _, err := runCommandWithOutput(cmd)
if err != nil {
t.Fatal(err)
}
runningContainerID := strings.TrimSpace(out)
containerResolvHashBefore, err = readContainerFile(runningContainerID, "resolv.conf.hash")
if err != nil {
t.Fatal(err)
}
// replace resolv.conf
if err := ioutil.WriteFile("/etc/resolv.conf", bytesResolvConf, 0644); err != nil {
t.Fatal(err)
}
// make sure the updater has time to run to validate we really aren't
// getting updated
time.Sleep(time.Second / 2)
containerResolvHashAfter, err = readContainerFile(runningContainerID, "resolv.conf.hash")
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(containerResolvHashBefore, containerResolvHashAfter) {
t.Fatalf("Running container's resolv.conf should not be updated; expected hash: %v, new hash: %v", containerResolvHashBefore, containerResolvHashAfter)
}
//4. test that a running container's resolv.conf is updated upon restart
// (the above container is still running..)
cmd = exec.Command(dockerBinary, "restart", runningContainerID)
if _, err = runCommand(cmd); err != nil {
t.Fatal(err)
}
// check for update in container
containerResolv, err = readContainerFile(runningContainerID, "resolv.conf")
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(containerResolv, bytesResolvConf) {
t.Fatalf("Restarted container should have updated resolv.conf; expected %q, got %q", tmpResolvConf, string(containerResolv))
}
//5. test that additions of a localhost resolver are cleaned from
// host resolv.conf before updating container's resolv.conf copies
// replace resolv.conf with a localhost-only nameserver copy
bytesResolvConf = []byte(tmpLocalhostResolvConf)
if err = ioutil.WriteFile("/etc/resolv.conf", bytesResolvConf, 0644); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second / 2)
// our first exited container ID should have been updated, but with default DNS
// after the cleanup of resolv.conf found only a localhost nameserver:
containerResolv, err = readContainerFile(containerID1, "resolv.conf")
if err != nil {
t.Fatal(err)
}
expected := "\nnameserver 8.8.8.8\nnameserver 8.8.4.4"
if !bytes.Equal(containerResolv, []byte(expected)) {
t.Fatalf("Container does not have cleaned/replaced DNS in resolv.conf; expected %q, got %q", expected, string(containerResolv))
}
//cleanup, restore original resolv.conf happens in defer func()
logDone("run - resolv.conf updater")
}
func TestRunAddHost(t *testing.T) {
defer deleteAllContainers()
cmd := exec.Command(dockerBinary, "run", "--add-host=extra:86.75.30.9", "busybox", "grep", "extra", "/etc/hosts")

Просмотреть файл

@ -5,13 +5,25 @@ import (
"io/ioutil"
"regexp"
"strings"
"sync"
log "github.com/Sirupsen/logrus"
"github.com/docker/docker/utils"
)
var (
nsRegexp = regexp.MustCompile(`^\s*nameserver\s*(([0-9]+\.){3}([0-9]+))\s*$`)
searchRegexp = regexp.MustCompile(`^\s*search\s*(([^\s]+\s*)*)$`)
defaultDns = []string{"8.8.8.8", "8.8.4.4"}
localHostRegexp = regexp.MustCompile(`(?m)^nameserver 127[^\n]+\n*`)
nsRegexp = regexp.MustCompile(`^\s*nameserver\s*(([0-9]+\.){3}([0-9]+))\s*$`)
searchRegexp = regexp.MustCompile(`^\s*search\s*(([^\s]+\s*)*)$`)
)
var lastModified struct {
sync.Mutex
sha256 string
contents []byte
}
func Get() ([]byte, error) {
resolv, err := ioutil.ReadFile("/etc/resolv.conf")
if err != nil {
@ -20,6 +32,57 @@ func Get() ([]byte, error) {
return resolv, nil
}
// Retrieves the host /etc/resolv.conf file, checks against the last hash
// and, if modified since last check, returns the bytes and new hash.
// This feature is used by the resolv.conf updater for containers
func GetIfChanged() ([]byte, string, error) {
lastModified.Lock()
defer lastModified.Unlock()
resolv, err := ioutil.ReadFile("/etc/resolv.conf")
if err != nil {
return nil, "", err
}
newHash, err := utils.HashData(bytes.NewReader(resolv))
if err != nil {
return nil, "", err
}
if lastModified.sha256 != newHash {
lastModified.sha256 = newHash
lastModified.contents = resolv
return resolv, newHash, nil
}
// nothing changed, so return no data
return nil, "", nil
}
// retrieve the last used contents and hash of the host resolv.conf
// Used by containers updating on restart
func GetLastModified() ([]byte, string) {
lastModified.Lock()
defer lastModified.Unlock()
return lastModified.contents, lastModified.sha256
}
// RemoveReplaceLocalDns looks for localhost (127.*) entries in the provided
// resolv.conf, removing local nameserver entries, and, if the resulting
// cleaned config has no defined nameservers left, adds default DNS entries
// It also returns a boolean to notify the caller if changes were made at all
func RemoveReplaceLocalDns(resolvConf []byte) ([]byte, bool) {
changed := false
cleanedResolvConf := localHostRegexp.ReplaceAll(resolvConf, []byte{})
// if the resulting resolvConf is empty, use defaultDns
if !bytes.Contains(cleanedResolvConf, []byte("nameserver")) {
log.Infof("No non-localhost DNS nameservers are left in resolv.conf. Using default external servers : %v", defaultDns)
cleanedResolvConf = append(cleanedResolvConf, []byte("\nnameserver "+strings.Join(defaultDns, "\nnameserver "))...)
}
if !bytes.Equal(resolvConf, cleanedResolvConf) {
changed = true
}
return cleanedResolvConf, changed
}
// getLines parses input into lines and strips away comments.
func getLines(input []byte, commentMarker []byte) [][]byte {
lines := bytes.Split(input, []byte("\n"))

Просмотреть файл

@ -156,3 +156,34 @@ func TestBuildWithZeroLengthDomainSearch(t *testing.T) {
t.Fatalf("Expected to not find '%s' got '%s'", notExpected, content)
}
}
func TestRemoveReplaceLocalDns(t *testing.T) {
ns0 := "nameserver 10.16.60.14\nnameserver 10.16.60.21\n"
if result, _ := RemoveReplaceLocalDns([]byte(ns0)); result != nil {
if ns0 != string(result) {
t.Fatalf("Failed No Localhost: expected \n<%s> got \n<%s>", ns0, string(result))
}
}
ns1 := "nameserver 10.16.60.14\nnameserver 10.16.60.21\nnameserver 127.0.0.1\n"
if result, _ := RemoveReplaceLocalDns([]byte(ns1)); result != nil {
if ns0 != string(result) {
t.Fatalf("Failed Localhost: expected \n<%s> got \n<%s>", ns0, string(result))
}
}
ns1 = "nameserver 10.16.60.14\nnameserver 127.0.0.1\nnameserver 10.16.60.21\n"
if result, _ := RemoveReplaceLocalDns([]byte(ns1)); result != nil {
if ns0 != string(result) {
t.Fatalf("Failed Localhost: expected \n<%s> got \n<%s>", ns0, string(result))
}
}
ns1 = "nameserver 127.0.1.1\nnameserver 10.16.60.14\nnameserver 10.16.60.21\n"
if result, _ := RemoveReplaceLocalDns([]byte(ns1)); result != nil {
if ns0 != string(result) {
t.Fatalf("Failed Localhost: expected \n<%s> got \n<%s>", ns0, string(result))
}
}
}

Просмотреть файл

@ -290,14 +290,6 @@ func NewHTTPRequestError(msg string, res *http.Response) error {
}
}
var localHostRx = regexp.MustCompile(`(?m)^nameserver 127[^\n]+\n*`)
// RemoveLocalDns looks into the /etc/resolv.conf,
// and removes any local nameserver entries.
func RemoveLocalDns(resolvConf []byte) []byte {
return localHostRx.ReplaceAll(resolvConf, []byte{})
}
// An StatusError reports an unsuccessful exit by a command.
type StatusError struct {
Status string