Do not read config in targetWorker, instead pass CacheOptions as part of worker data (#667)

* remove getConfig call from targetWorker

* Remove custom storage config from cache options

* Fix

* Change files

* Update change file

* Fix tests

change/@lage-run-cli-0f80a060-12b0-4906-9385-1578ca623068.json

@@ -0,0 +1,7 @@
+{
+  "type": "patch",
+  "comment": "Do not read config in targetWorker, instead pass CacheOptions as part of workerdata",
+  "packageName": "@lage-run/cli",
+  "email": "altinokd@microsoft.com",
+  "dependentChangeType": "patch"
+}

change/@lage-run-config-65433d34-146b-43b8-a244-9197537a7bc2.json

@@ -0,0 +1,7 @@
+{
+  "type": "minor",
+  "comment": "Do not read config in targetWorker, instead pass CacheOptions as part of workerdata",
+  "packageName": "@lage-run/config",
+  "email": "altinokd@microsoft.com",
+  "dependentChangeType": "patch"
+}

change/@lage-run-scheduler-35753a2a-43bc-47ee-8f4c-db297344e84c.json

@@ -0,0 +1,7 @@
+{
+  "type": "patch",
+  "comment": "Do not read config in targetWorker, instead pass CacheOptions as part of workerdata",
+  "packageName": "@lage-run/scheduler",
+  "email": "altinokd@microsoft.com",
+  "dependentChangeType": "patch"
+}

packages/cli/src/commands/run/runAction.ts

@@ -90,6 +90,7 @@ export async function runAction(options: RunOptions, command: Command) {
       root,
       taskArgs,
       skipLocalCache: options.skipLocalCache,
+      cacheOptions: config.cacheOptions,
       runners: {
         npmScript: {
           script: require.resolve("./runners/NpmScriptRunner.js"),

packages/cli/src/commands/run/watchAction.ts

@@ -86,6 +86,7 @@ export async function watchAction(options: RunOptions, command: Command) {
       root,
       taskArgs,
       skipLocalCache: options.skipLocalCache,
+      cacheOptions: config.cacheOptions,
       runners: {
         npmScript: {
           script: require.resolve("./runners/NpmScriptRunner.js"),

packages/config/src/types/CacheOptions.ts

@@ -1,6 +1,10 @@
-import type { Config as BackfillCacheOptions } from "backfill-config";
+import type { Config as BackfillCacheOptions, CustomStorageConfig } from "backfill-config";
 
-export type CacheOptions = BackfillCacheOptions & {
+export type LageBackfillCacheOptions = Omit<BackfillCacheOptions, "cacheStorageConfig"> & {
+  cacheStorageConfig: Exclude<BackfillCacheOptions["cacheStorageConfig"], CustomStorageConfig>;
+};
+
+export type CacheOptions = LageBackfillCacheOptions & {
   environmentGlob: string[];
   cacheKey: string;
   writeRemoteCache: boolean;

packages/e2e-tests/src/remoteFallback.test.ts

@@ -18,15 +18,9 @@ describe("RemoteFallbackCacheProvider", () => {
         cacheOptions: {
           writeRemoteCache: true,
           cacheStorageConfig: {
-            provider: (logger, cwd) => ({
-              async fetch(hash) {
-                return false;
-              },
-
-              async put(hash, filesToCache) {
-              },
-            }),
+            provider: 'local'
           },
+          internalCacheFolder: '.lage-cache-test'
         } 
       };`
     );
@@ -115,16 +109,10 @@ describe("RemoteFallbackCacheProvider", () => {
         cache: true,
         cacheOptions: {
           cacheStorageConfig: {
-            provider: (logger, cwd) => ({
-              async fetch(hash) {
-                return false;
-              },
-
-              async put(hash, filesToCache) {
-              },
-            }),
+            provider: 'local'
           },
-        } 
+          internalCacheFolder: '.lage-cache-test'
+        }  
       };`
     );
 
@@ -169,16 +157,10 @@ describe("RemoteFallbackCacheProvider", () => {
         cacheOptions: {
           writeRemoteCache: true,
           cacheStorageConfig: {
-            provider: (logger, cwd) => ({
-              async fetch(hash) {
-                return false;
-              },
-
-              async put(hash, filesToCache) {
-              },
-            }),
+            provider: 'local'
           },
-        } 
+          internalCacheFolder: '.lage-cache-test'
+        }
       };`
     );
 

packages/scheduler/src/SimpleScheduler.ts

@@ -11,6 +11,7 @@ import type { TargetScheduler, SchedulerRunResults, SchedulerRunSummary, TargetR
 import type { Pool } from "@lage-run/worker-threads-pool";
 import type { TargetRunnerPickerOptions } from "@lage-run/scheduler-types";
 import type { TargetHasher } from "@lage-run/hasher";
+import type { CacheOptions } from "@lage-run/cache";
 
 export interface SimpleSchedulerOptions {
   logger: Logger;
@@ -23,6 +24,7 @@ export interface SimpleSchedulerOptions {
     root: string;
     taskArgs: string[];
     skipLocalCache?: boolean;
+    cacheOptions?: CacheOptions;
   };
   maxWorkersPerTask: Map<string, number>;
   pool?: Pool; // for testing

packages/scheduler/src/workers/targetWorker.ts

@@ -1,10 +1,10 @@
 import { createCache } from "../cache/createCacheProvider.js";
-import { getConfig } from "@lage-run/config";
 import { registerWorker } from "@lage-run/worker-threads-pool";
 import { TargetRunnerPicker } from "../runners/TargetRunnerPicker.js";
 import { parentPort, workerData } from "worker_threads";
 import createLogger from "@lage-run/logger";
 
+import type { CacheOptions } from "@lage-run/config";
 import type { Target } from "@lage-run/target-graph";
 import type { TargetRunnerPickerOptions } from "@lage-run/scheduler-types";
 
@@ -15,12 +15,11 @@ interface TargetWorkerDataOptions {
   shouldResetCache: boolean;
   taskArgs: string[];
   root: string;
+  cacheOptions?: CacheOptions;
 }
 
 async function setup(options: TargetWorkerDataOptions) {
-  const { runners, root } = options;
-
-  const config = await getConfig(root);
+  const { runners, root, cacheOptions } = options;
 
   const logger = createLogger();
   logger.addReporter({
@@ -33,7 +32,7 @@ async function setup(options: TargetWorkerDataOptions) {
   const { cacheProvider } = await createCache({
     root,
     logger,
-    cacheOptions: config.cacheOptions,
+    cacheOptions,
     cliArgs: options.taskArgs,
     skipLocalCache: options.skipLocalCache,
   });