diff --git a/middleware/handle-invalid-paths.js b/middleware/handle-invalid-paths.js
index 7d50be7a7a..2a8acda843 100644
--- a/middleware/handle-invalid-paths.js
+++ b/middleware/handle-invalid-paths.js
@@ -1,6 +1,32 @@
-export default function handleInvalidPaths(req, res, next) {
+import { defaultCacheControl } from './cache-control.js'
+
+const JUNK_PATHS = new Set([
+  '/.env',
+  '/env',
+  '/xmlrpc.php',
+  '/wp-login.php',
+  '/README.md',
+  '/server.js',
+  '/package.json',
+  '/.git',
+])
+
+function isJunkPath(path) {
+  if (JUNK_PATHS.has(path)) return true
+
   // Prevent various malicious injection attacks targeting Next.js
-  if (req.path.match(/^\/_next[^/]/) || req.path === '/_next/data' || req.path === '/_next/data/') {
+  if (path.match(/^\/_next[^/]/) || path === '/_next/data' || path === '/_next/data/') {
+    return true
+  }
+
+  return false
+}
+
+export default function handleInvalidPaths(req, res, next) {
+  if (isJunkPath(req.path)) {
+    // We can all the CDN to cache these responses because they're
+    // they're not going to suddenly work in the next deployment.
+    defaultCacheControl(res)
     res.setHeader('content-type', 'text/plain')
     return res.status(404).send('Not found')
   }
diff --git a/tests/rendering/server.js b/tests/rendering/server.js
index 927d2763f8..abf8d5c053 100644
--- a/tests/rendering/server.js
+++ b/tests/rendering/server.js
@@ -399,9 +399,27 @@ describe('static routes', () => {
     expect((await get('/public/ghae/schema.docs-ghae.graphql')).statusCode).toBe(200)
   })
 
-  it('does not serve repo contents that live outside the /assets directory', async () => {
-    expect((await get('/package.json', { followRedirects: true })).statusCode).toBe(404)
-    expect((await get('/README.md', { followRedirects: true })).statusCode).toBe(404)
-    expect((await get('/server.js', { followRedirects: true })).statusCode).toBe(404)
+  test('does not serve repo contents that live outside the /assets directory', async () => {
+    const paths = ['/package.json', '/README.md', '/server.js', '/.git', '/.env']
+    for (const path of paths) {
+      const res = await get(path)
+      expect(res.statusCode).toBe(404)
+      expect(res.headers['content-type']).toMatch('text/plain')
+      expect(res.headers['cache-control']).toMatch(/max-age=[1-9]/)
+      expect(res.headers['cache-control']).toMatch('public')
+    }
+    expect.assertions(4 * paths.length)
+  })
+
+  test('junk requests with or without query strings is 404', async () => {
+    const paths = ['/env', '/xmlrpc.php', '/wp-login.php']
+    for (const path of paths) {
+      const res = await get(`${path}?r=${Math.random()}`)
+      expect(res.statusCode).toBe(404)
+      expect(res.headers['content-type']).toMatch('text/plain')
+      expect(res.headers['cache-control']).toMatch(/max-age=[1-9]/)
+      expect(res.headers['cache-control']).toMatch('public')
+    }
+    expect.assertions(4 * paths.length)
   })
 })