diff --git a/builtin/apply.c b/builtin/apply.c
index 771c972c55..660cf92538 100644
--- a/builtin/apply.c
+++ b/builtin/apply.c
@@ -3141,11 +3141,7 @@ static void remove_file(struct patch *patch, int rmdir_empty)
 			die("unable to remove %s from index", patch->old_name);
 	}
 	if (!cached) {
-		if (S_ISGITLINK(patch->old_mode)) {
-			if (rmdir(patch->old_name))
-				warning("unable to remove submodule %s",
-					patch->old_name);
-		} else if (!unlink_or_warn(patch->old_name) && rmdir_empty) {
+		if (!remove_or_warn(patch->old_mode, patch->old_name) && rmdir_empty) {
 			remove_path(patch->old_name);
 		}
 	}
diff --git a/git-compat-util.h b/git-compat-util.h
index 7e62b55270..6e25977289 100644
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@ -479,5 +479,14 @@ void git_qsort(void *base, size_t nmemb, size_t size,
  * Always returns the return value of unlink(2).
  */
 int unlink_or_warn(const char *path);
+/*
+ * Likewise for rmdir(2).
+ */
+int rmdir_or_warn(const char *path);
+/*
+ * Calls the correct function out of {unlink,rmdir}_or_warn based on
+ * the supplied file mode.
+ */
+int remove_or_warn(unsigned int mode, const char *path);
 
 #endif
diff --git a/merge-recursive.c b/merge-recursive.c
index 917397ca7a..206c103635 100644
--- a/merge-recursive.c
+++ b/merge-recursive.c
@@ -409,7 +409,7 @@ static int remove_file(struct merge_options *o, int clean,
 			return -1;
 	}
 	if (update_working_directory) {
-		if (remove_path(path) && errno != ENOENT)
+		if (remove_path(path))
 			return -1;
 	}
 	return 0;
diff --git a/t/t4134-apply-submodule.sh b/t/t4134-apply-submodule.sh
new file mode 100755
index 0000000000..1b82f93cff
--- /dev/null
+++ b/t/t4134-apply-submodule.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+#
+# Copyright (c) 2010 Peter Collingbourne
+#
+
+test_description='git apply submodule tests'
+
+. ./test-lib.sh
+
+test_expect_success setup '
+	cat > create-sm.patch <<EOF
+diff --git a/dir/sm b/dir/sm
+new file mode 160000
+index 0000000..0123456
+--- /dev/null
++++ b/dir/sm
+@@ -0,0 +1 @@
++Subproject commit 0123456789abcdef0123456789abcdef01234567
+EOF
+	cat > remove-sm.patch <<EOF
+diff --git a/dir/sm b/dir/sm
+deleted file mode 160000
+index 0123456..0000000
+--- a/dir/sm
++++ /dev/null
+@@ -1 +0,0 @@
+-Subproject commit 0123456789abcdef0123456789abcdef01234567
+EOF
+'
+
+test_expect_success 'removing a submodule also removes all leading subdirectories' '
+	git apply --index create-sm.patch &&
+	test -d dir/sm &&
+	git apply --index remove-sm.patch &&
+	test \! -d dir
+'
+
+test_done
diff --git a/unpack-trees.c b/unpack-trees.c
index 75f54cac97..c29a9e067f 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -67,16 +67,8 @@ static void unlink_entry(struct cache_entry *ce)
 {
 	if (has_symlink_or_noent_leading_path(ce->name, ce_namelen(ce)))
 		return;
-	if (S_ISGITLINK(ce->ce_mode)) {
-		if (rmdir(ce->name)) {
-			warning("unable to rmdir %s: %s",
-				ce->name, strerror(errno));
-			return;
-		}
-	}
-	else
-		if (unlink_or_warn(ce->name))
-			return;
+	if (remove_or_warn(ce->ce_mode, ce->name))
+		return;
 	schedule_dir_for_removal(ce->name, ce_namelen(ce));
 }
 
diff --git a/wrapper.c b/wrapper.c
index 9c71b21242..10a6750795 100644
--- a/wrapper.c
+++ b/wrapper.c
@@ -311,18 +311,30 @@ int odb_pack_keep(char *name, size_t namesz, unsigned char *sha1)
 	return open(name, O_RDWR|O_CREAT|O_EXCL, 0600);
 }
 
-int unlink_or_warn(const char *file)
+static int warn_if_unremovable(const char *op, const char *file, int rc)
 {
-	int rc = unlink(file);
-
 	if (rc < 0) {
 		int err = errno;
 		if (ENOENT != err) {
-			warning("unable to unlink %s: %s",
-				file, strerror(errno));
+			warning("unable to %s %s: %s",
+				op, file, strerror(errno));
 			errno = err;
 		}
 	}
 	return rc;
 }
 
+int unlink_or_warn(const char *file)
+{
+	return warn_if_unremovable("unlink", file, unlink(file));
+}
+
+int rmdir_or_warn(const char *file)
+{
+	return warn_if_unremovable("rmdir", file, rmdir(file));
+}
+
+int remove_or_warn(unsigned int mode, const char *file)
+{
+	return S_ISGITLINK(mode) ? rmdir_or_warn(file) : unlink_or_warn(file);
+}