use gitignore to get files to hash instead of manually specified globs

README.md

@@ -95,7 +95,6 @@ The default configuration object is:
 {
   cacheStorageConfig: { provider: "local" },
   clearOutputFolder: false,
-  hashGlobs: ["**/*", "!**/node_modules/**", "!lib/**", "!**/tsconfig.tsbuildinfo"],
   internalCacheFolder: "node_modules/.cache/backfill",
   logFolder: "node_modules/.cache/backfill",
   logLevel: "info",
@@ -120,7 +119,6 @@ The configuration type is:
 export type Config = {
   cacheStorageConfig: CacheStorageConfig;
   clearOutputFolder: boolean;
-  hashGlobs: HashGlobs;
   internalCacheFolder: string;
   logFolder: string;
   logLevel: LogLevels;

packages/backfill/src/__tests__/e2e.test.ts

@@ -21,7 +21,7 @@ describe("End to end", () => {
 
     // Verify it produces the correct hash
     const ownHash = fs.readdirSync(path.join(packageRoot, hashPath));
-    expect(ownHash).toContain("57f26541cc848f71a80fd9039137f1d50e013b92");
+    expect(ownHash).toContain("88e8e1578cb7fd86d6162025488b985a49ee1482");
 
     // ... and that `npm run compile` was run successfully
     const libFolderExist = await fs.pathExists("lib");

packages/backfill/src/api.ts

@@ -38,8 +38,8 @@ export async function computeHash(
   logger: Logger,
   hashSalt?: string
 ): Promise<string> {
-  const { outputGlob, packageRoot, hashGlobs } = createConfig(logger, cwd);
-  const hasher = new Hasher({ packageRoot, outputGlob, hashGlobs }, logger);
+  const { packageRoot } = createConfig(logger, cwd);
+  const hasher = new Hasher({ packageRoot }, logger);
 
   const hash = await hasher.createPackageHash(hashSalt || "");
   return hash;
@@ -49,8 +49,8 @@ export async function computeHashOfOutput(
   cwd: string,
   logger: Logger
 ): Promise<string> {
-  const { outputGlob, packageRoot, hashGlobs } = createConfig(logger, cwd);
-  const hasher = new Hasher({ packageRoot, outputGlob, hashGlobs }, logger);
+  const { packageRoot } = createConfig(logger, cwd);
+  const hasher = new Hasher({ packageRoot }, logger);
   const hash = await hasher.hashOfOutput();
   return hash;
 }

packages/backfill/src/audit.ts

@@ -36,7 +36,6 @@ export function initializeWatcher(
   internalCacheFolder: string,
   logFolder: string,
   outputGlob: string[],
-  hashGlobs: string[],
   logger: Logger
 ) {
   // Trying to find the git root and using it as an approximation of code boundary
@@ -58,7 +57,7 @@ export function initializeWatcher(
     internalCacheFolder
   ]);
   watcher = chokidar
-    .watch(hashGlobs, {
+    .watch(["**"], {
       ignored: ignoreGlobs,
       cwd: repositoryRoot,
       persistent: true,

packages/backfill/src/index.ts

@@ -113,7 +113,6 @@ export async function main(): Promise<void> {
     const config = createConfig(logger, cwd);
     const {
       clearOutput,
-      hashGlobs,
       internalCacheFolder,
       logFolder,
       logLevel,
@@ -150,7 +149,6 @@ export async function main(): Promise<void> {
         internalCacheFolder,
         logFolder,
         outputGlob,
-        hashGlobs,
         logger
       );
     }

packages/config/src/index.ts

@@ -10,8 +10,6 @@ import { getEnvConfig } from "./envConfig";
 export * from "./cacheConfig";
 export * from "./envConfig";
 
-export type HashGlobs = string[];
-
 export const modesObject = {
   READ_ONLY: "",
   WRITE_ONLY: "",
@@ -24,7 +22,6 @@ export type BackfillModes = keyof typeof modesObject;
 export type Config = {
   cacheStorageConfig: CacheStorageConfig;
   clearOutput: boolean;
-  hashGlobs: HashGlobs;
   internalCacheFolder: string;
   logFolder: string;
   logLevel: LogLevel;
@@ -81,12 +78,6 @@ export function createDefaultConfig(fromPath: string): Config {
       provider: "local"
     },
     clearOutput: false,
-    hashGlobs: [
-      "**/*",
-      "!**/node_modules/**",
-      `!lib/**`,
-      "!**/tsconfig.tsbuildinfo"
-    ],
     internalCacheFolder: defaultCacheFolder,
     logFolder: defaultCacheFolder,
     logLevel: "info",

packages/hasher/package.json

@@ -20,6 +20,7 @@
     "backfill-config": "^4.0.0",
     "backfill-logger": "^4.0.0",
     "fast-glob": "^3.0.4",
+    "globby": "^11.0.0",
     "find-up": "^4.1.0",
     "find-yarn-workspace-root": "^1.2.1",
     "fs-extra": "^8.1.0"

packages/hasher/src/__tests__/hashOfFiles.test.ts

@@ -5,47 +5,22 @@ import { setupFixture } from "backfill-utils-test";
 import { generateHashOfFiles } from "../hashOfFiles";
 
 describe("generateHashOfFiles()", () => {
-  it("excludes files provided by backfill config", async () => {
-    const packageRoot = await setupFixture("monorepo");
-
-    const hashOfEverything = await generateHashOfFiles(packageRoot, ["**"]);
-
-    const hashExcludeNodeModules = await generateHashOfFiles(packageRoot, [
-      "**",
-      "!**/node_modules/**"
-    ]);
-
-    expect(hashOfEverything).not.toEqual(hashExcludeNodeModules);
-  });
-
   it("creates different hashes for different hashes", async () => {
     const packageRoot = await setupFixture("monorepo");
 
-    const hashOfPackage = await generateHashOfFiles(packageRoot, [
-      "**",
-      "!**/node_modules/**"
-    ]);
+    const hashOfPackage = await generateHashOfFiles(packageRoot);
 
     fs.writeFileSync("foo.txt", "bar");
-    const hashOfPackageWithFoo = await generateHashOfFiles(packageRoot, [
-      "**",
-      "!**/node_modules/**"
-    ]);
+    const hashOfPackageWithFoo = await generateHashOfFiles(packageRoot);
     expect(hashOfPackage).not.toEqual(hashOfPackageWithFoo);
 
     fs.writeFileSync("foo.txt", "foo");
-    const hashOfPackageWithFoo2 = await generateHashOfFiles(packageRoot, [
-      "**",
-      "!**/node_modules/**"
-    ]);
+    const hashOfPackageWithFoo2 = await generateHashOfFiles(packageRoot);
     expect(hashOfPackageWithFoo).not.toEqual(hashOfPackageWithFoo2);
 
     fs.unlinkSync("foo.txt");
 
-    const hashOfPackageWithoutFoo = await generateHashOfFiles(packageRoot, [
-      "**",
-      "!**/node_modules/**"
-    ]);
+    const hashOfPackageWithoutFoo = await generateHashOfFiles(packageRoot);
     expect(hashOfPackage).toEqual(hashOfPackageWithoutFoo);
   });
 });

packages/hasher/src/hashOfFiles.ts

@@ -1,7 +1,8 @@
 import crypto from "crypto";
 import path from "path";
-import fg from "fast-glob";
+import globby from "globby";
 import fs from "fs-extra";
+import findUp from "find-up";
 
 import { hashStrings } from "./helpers";
 
@@ -9,27 +10,40 @@ const newline = /\r\n|\r|\n/g;
 const LF = "\n";
 
 export async function generateHashOfFiles(
-  packageRoot: string,
-  globs: string[]
+  packageRoot: string
 ): Promise<string> {
-  const files = await fg(globs, {
+  const nearestGitFolder = await findUp(".git", {
     cwd: packageRoot,
-    onlyFiles: false,
-    objectMode: true
+    type: "directory"
   });
 
-  files.sort((a, b) => a.path.localeCompare(b.path));
+  if (nearestGitFolder === undefined) {
+    throw new Error(
+      `It does not seem that this package is in a git repo: ${packageRoot}`
+    );
+  }
+
+  const repoRoot = path.dirname(nearestGitFolder);
+
+  // If the package is a git repo by itself then the search pattern is all the files
+  const relativePackagePath = path.relative(repoRoot, packageRoot) || "**/*";
+
+  // Note: globby does not support objectMode
+  const files = await globby(relativePackagePath, {
+    cwd: repoRoot,
+    gitignore: true
+  });
+
+  files.sort((a, b) => a.localeCompare(b));
 
   const hashes = await Promise.all(
     files.map(async file => {
       const hasher = crypto.createHash("sha1");
-      hasher.update(file.path);
+      hasher.update(file);
 
-      if (!file.dirent.isDirectory()) {
-        const fileBuffer = await fs.readFile(path.join(packageRoot, file.path));
-        const data = fileBuffer.toString().replace(newline, LF);
-        hasher.update(data);
-      }
+      const fileBuffer = await fs.readFile(path.join(repoRoot, file));
+      const data = fileBuffer.toString().replace(newline, LF);
+      hasher.update(data);
 
       return hasher.digest("hex");
     })

packages/hasher/src/hashOfPackage.ts

@@ -42,8 +42,7 @@ export async function getPackageHash(
   packageRoot: string,
   workspaces: WorkspaceInfo,
   yarnLock: ParsedYarnLock,
-  logger: Logger,
-  hashGlobs: string[]
+  logger: Logger
 ): Promise<PackageHashInfo> {
   const memoizationKey = path.resolve(packageRoot);
 
@@ -77,7 +76,7 @@ export async function getPackageHash(
     ...externalDeoendencies
   ];
 
-  const filesHash = await generateHashOfFiles(packageRoot, hashGlobs);
+  const filesHash = await generateHashOfFiles(packageRoot);
   const dependenciesHash = hashStrings(resolvedDependencies);
 
   logger.silly(name);

packages/hasher/src/index.ts

@@ -46,20 +46,14 @@ export function addToQueue(
 
 export class Hasher implements IHasher {
   private packageRoot: string;
-  private outputGlob: string[];
-  private hashGlobs: string[];
 
   constructor(
     private options: {
       packageRoot: string;
-      outputGlob: string[];
-      hashGlobs: string[];
     },
     private logger: Logger
   ) {
     this.packageRoot = this.options.packageRoot;
-    this.outputGlob = this.options.outputGlob;
-    this.hashGlobs = this.options.hashGlobs;
   }
 
   public async createPackageHash(salt: string): Promise<string> {
@@ -83,8 +77,7 @@ export class Hasher implements IHasher {
         packageRoot,
         workspaces,
         yarnLock,
-        this.logger,
-        this.hashGlobs
+        this.logger
       );
 
       addToQueue(packageHash.internalDependencies, queue, done, workspaces);
@@ -107,6 +100,6 @@ export class Hasher implements IHasher {
   }
 
   public async hashOfOutput(): Promise<string> {
-    return generateHashOfFiles(this.packageRoot, this.outputGlob);
+    return generateHashOfFiles(this.packageRoot);
   }
 }

packages/utils-test/src/setupFixture.ts

@@ -1,7 +1,9 @@
 import path from "path";
+import { EOL } from "os";
 import findUp from "find-up";
 import fs from "fs-extra";
 import tempy from "tempy";
+import execa from "execa";
 
 async function findFixturePath(cwd: string, fixtureName: string) {
   return await findUp(path.join("__fixtures__", fixtureName), {
@@ -29,5 +31,8 @@ export async function setupFixture(fixtureName: string) {
   process.chdir(cwd);
   fs.copySync(fixturePath, cwd);
 
+  await execa("git", ["init"]);
+  fs.writeFileSync(path.join(cwd, ".gitignore"), `node_modules${EOL}lib`);
+
   return cwd;
 }

yarn.lock

@@ -3352,6 +3352,18 @@ fast-glob@^3.0.3, fast-glob@^3.0.4:
     merge2 "^1.3.0"
     micromatch "^4.0.2"
 
+fast-glob@^3.1.1:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.2.tgz#ade1a9d91148965d4bf7c51f72e1ca662d32e63d"
+  integrity sha512-UDV82o4uQyljznxwMxyVRJgZZt3O5wENYojjzbaGEGZgeOxkLFf+V4cnUD+krzb2F72E18RhamkMZ7AdeggF7A==
+  dependencies:
+    "@nodelib/fs.stat" "^2.0.2"
+    "@nodelib/fs.walk" "^1.2.3"
+    glob-parent "^5.1.0"
+    merge2 "^1.3.0"
+    micromatch "^4.0.2"
+    picomatch "^2.2.1"
+
 fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2"
@@ -3819,6 +3831,18 @@ globby@^10.0.1:
     merge2 "^1.2.3"
     slash "^3.0.0"
 
+globby@^11.0.0:
+  version "11.0.0"
+  resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.0.tgz#56fd0e9f0d4f8fb0c456f1ab0dee96e1380bc154"
+  integrity sha512-iuehFnR3xu5wBBtm4xi0dMe92Ob87ufyu/dHwpDYfbcpYpIbrO5OnS8M1vWvrBhSGEJ3/Ecj7gnX76P8YxpPEg==
+  dependencies:
+    array-union "^2.1.0"
+    dir-glob "^3.0.1"
+    fast-glob "^3.1.1"
+    ignore "^5.1.4"
+    merge2 "^1.3.0"
+    slash "^3.0.0"
+
 globby@^9.2.0:
   version "9.2.0"
   resolved "https://registry.yarnpkg.com/globby/-/globby-9.2.0.tgz#fd029a706c703d29bdd170f4b6db3a3f7a7cb63d"
@@ -4031,7 +4055,7 @@ ignore@^4.0.3, ignore@^4.0.6:
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc"
   integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==
 
-ignore@^5.1.1:
+ignore@^5.1.1, ignore@^5.1.4:
   version "5.1.4"
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.4.tgz#84b7b3dbe64552b6ef0eca99f6743dbec6d97adf"
   integrity sha512-MzbUSahkTW1u7JpKKjY7LCARd1fU5W2rLdxlM4kdkayuCwZImjkpluF9CM1aLewYJguPDqewLam18Y6AU69A8A==
@@ -6394,6 +6418,11 @@ picomatch@^2.0.7:
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.1.1.tgz#ecdfbea7704adb5fe6fb47f9866c4c0e15e905c5"
   integrity sha512-OYMyqkKzK7blWO/+XZYP6w8hH0LDvkBvdvKukti+7kqYFCiEAk+gI3DWnryapc0Dau05ugGTy0foQ6mqn4AHYA==
 
+picomatch@^2.2.1:
+  version "2.2.2"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad"
+  integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==
+
 pify@^2.0.0, pify@^2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"