Support custom requirements.txt location (#1219)

* support custom requirements.txt location

* add more tests

Co-authored-by: Stella Qian <zixuan.qian@microsoft.com>

doc/configuration.md

@@ -47,7 +47,8 @@ Setting name for Python apps | Description
 PYTHON\_VERSION              | Specify which Python version the app is using                  | ""      | "2.7.1"
 DISABLE\_PYTHON\_BUILD       | Do not apply Python build even if repo indicates it            | `false` | `true`, `false`
 VIRTUALENV\_NAME             | Specify Python virtual environment name                        | ""      | "antenv2.7"
-DISABLE\_COLLECTSTATIC       | Disable running `collectstatic` when building Django apps.      | `false` | `true`, `false`
+DISABLE\_COLLECTSTATIC       | Disable running `collectstatic` when building Django apps.     | `false` | `true`, `false`
+CUSTOM\_REQUIREMENTSTXT\_PATH| Specify where a requirements.txt is locating. If not set, default is at root of the repo.| ""    | "subdir/requirements.txt"
 
 Setting name for Php apps    | Description                                                    | Default | Example
 -----------------------------|----------------------------------------------------------------|---------|----------------

doc/runtimes/python.md

@@ -46,6 +46,7 @@ The Python toolset is run when the following conditions are met:
 1. `requirements.txt` in root of repo
 2. `runtime.txt` in root of repo
 3. Files with `.py` extension in root of repo or in sub-directories if set `DISABLE_RECURSIVE_LOOKUP=false`.
+4. `requirements.txt` at specific path within the repo if set `CUSTOM_REQUIREMENTSTXT_PATH`.
 
 ## Detect Conda environment and Python JupyterNotebook
 
@@ -61,7 +62,7 @@ The following process is applied for each build.
 1. Run custom script if specified by `PRE_BUILD_SCRIPT_PATH`.
 2. Create python virtual environment if specified by `VIRTUALENV_NAME`.
 3. Run `python -m pip install --cache-dir /usr/local/share/pip-cache --prefer-binary -r requirements.txt` 
-   if `requirements.txt` exists.
+   if `requirements.txt` exists in the root of repo or specified by `CUSTOM_REQUIREMENTSTXT_PATH`.
 4. Run `python setup.py install` if `setup.py` exists.
 5. Run python package commands and Determine python package wheel.
 6. If `manage.py` is found in the root of the repo `manage.py collectstatic` is run. However,
@@ -74,8 +75,8 @@ The following process is applied for each build.
 The following process is applied for each build.
 1. Run custom script if specified by `PRE_BUILD_SCRIPT_PATH`.
 2. Set up Conda virtual environemnt `conda env create --file $envFile`.
-3. If `requirment.txt` exists, activate environemnt `conda activate $environmentPrefix` and
-   run `pip install --no-cache-dir -r requirements.txt`.
+3. If `requirment.txt` exists in the root of repo or specified by `CUSTOM_REQUIREMENTSTXT_PATH`, activate environemnt 
+  `conda activate $environmentPrefix` and run `pip install --no-cache-dir -r requirements.txt`.
 4. Run custom script if specified by `POST_BUILD_SCRIPT_PATH`.
 
 

src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs

@@ -70,5 +70,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator
         public string BuildCommandsFileName { get; set; }
 
         public bool CompressDestinationDir { get; set; }
+        
+        public string CustomRequirementsTxtPath { get; set; }
     }
 }

src/BuildScriptGenerator/Python/PythonBashBuildSnippet.sh.tpl

@@ -23,6 +23,12 @@ if [ ! -d "$PIP_CACHE_DIR" ];then
     mkdir -p $PIP_CACHE_DIR
 fi
 
+{{ if CustomRequirementsTxtPath | IsNotBlank }}
+    REQUIREMENTS_TXT_FILE="{{ CustomRequirementsTxtPath }}"
+{{ else }}
+    REQUIREMENTS_TXT_FILE="requirements.txt"
+{{ end }}
+
 {{ if VirtualEnvironmentName | IsNotBlank }}
     {{ if PackagesDirectory | IsNotBlank }}
         if [ -d "{{ PackagesDirectory }}" ]
@@ -38,7 +44,7 @@ fi
 
     echo "Python Virtual Environment: $VIRTUALENVIRONMENTNAME"
 
-    if [ -e "requirements.txt" ]; then
+    if [ -e "$REQUIREMENTS_TXT_FILE" ]; then
         VIRTUALENVIRONMENTOPTIONS="$VIRTUALENVIRONMENTOPTIONS --system-site-packages"
     fi
 
@@ -55,13 +61,13 @@ fi
     source $VIRTUALENVIRONMENTNAME/bin/activate
 
     moreInformation="More information: https://aka.ms/troubleshoot-python"
-    if [ -e "requirements.txt" ]
+    if [ -e "$REQUIREMENTS_TXT_FILE" ]
     then
         set +e 
         echo "Running pip install..."
-        InstallCommand="python -m pip install --cache-dir $PIP_CACHE_DIR --prefer-binary -r requirements.txt | ts $TS_FMT"
+        InstallCommand="python -m pip install --cache-dir $PIP_CACHE_DIR --prefer-binary -r $REQUIREMENTS_TXT_FILE | ts $TS_FMT"
         printf %s " , $InstallCommand" >> "$COMMAND_MANIFEST_FILE"
-        StdError=$( ( python -m pip install --cache-dir $PIP_CACHE_DIR --prefer-binary -r requirements.txt | ts $TS_FMT; exit ${PIPESTATUS[0]} ) 2>&1; exit ${PIPESTATUS[0]} )
+        StdError=$( ( python -m pip install --cache-dir $PIP_CACHE_DIR --prefer-binary -r $REQUIREMENTS_TXT_FILE | ts $TS_FMT; exit ${PIPESTATUS[0]} ) 2>&1; exit ${PIPESTATUS[0]} )
         pipInstallExitCode=${PIPESTATUS[0]}
         set -e
         if [[ $pipInstallExitCode != 0 ]]
@@ -110,15 +116,15 @@ fi
     python_bin=python
 {{ else }}
     moreInformation="More information: https://aka.ms/troubleshoot-python"
-    if [ -e "requirements.txt" ]
+    if [ -e "$REQUIREMENTS_TXT_FILE" ]
     then
         set +e
         echo
         echo Running pip install...
         START_TIME=$SECONDS
-        InstallCommand="$python -m pip install --cache-dir $PIP_CACHE_DIR --prefer-binary -r requirements.txt --target="{{ PackagesDirectory }}" --upgrade | ts $TS_FMT"
+        InstallCommand="$python -m pip install --cache-dir $PIP_CACHE_DIR --prefer-binary -r $REQUIREMENTS_TXT_FILE --target="{{ PackagesDirectory }}" --upgrade | ts $TS_FMT"
         printf %s " , $InstallCommand" >> "$COMMAND_MANIFEST_FILE"
-        StdError=$( ( $python -m pip install --cache-dir $PIP_CACHE_DIR --prefer-binary -r requirements.txt --target="{{ PackagesDirectory }}" --upgrade | ts $TS_FMT; exit ${PIPESTATUS[0]} ) 2>&1; exit ${PIPESTATUS[0]} )
+        StdError=$( ( $python -m pip install --cache-dir $PIP_CACHE_DIR --prefer-binary -r $REQUIREMENTS_TXT_FILE --target="{{ PackagesDirectory }}" --upgrade | ts $TS_FMT; exit ${PIPESTATUS[0]} ) 2>&1; exit ${PIPESTATUS[0]} )
         pipInstallExitCode=${PIPESTATUS[0]}
         ELAPSED_TIME=$(($SECONDS - $START_TIME))
         echo "Done in $ELAPSED_TIME sec(s)."
@@ -221,7 +227,7 @@ fi
 {{ if EnableCollectStatic }}
     if [ -e "$SOURCE_DIR/manage.py" ]
     then
-        if grep -iq "Django" "$SOURCE_DIR/requirements.txt"
+        if grep -iq "Django" "$SOURCE_DIR/$REQUIREMENTS_TXT_FILE"
         then
             echo
             echo Content in source directory is a Django app
@@ -237,7 +243,7 @@ fi
             ELAPSED_TIME=$(($SECONDS - $START_TIME))
             echo "Done in $ELAPSED_TIME sec(s)."
         else
-            LogWarning "Missing Django module in $SOURCE_DIR/requirements.txt. Add Django to your requirements.txt file."
+            LogWarning "Missing Django module in $SOURCE_DIR/$REQUIREMENTS_TXT_FILE. Add Django to your requirements.txt file."
         fi
     fi
 {{ end }}

src/BuildScriptGenerator/Python/PythonBashBuildSnippetProperties.cs

@@ -23,7 +23,8 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
             bool runPythonPackageCommand,
             string pythonVersion,
             string pythonBuildCommandsFileName = null,
-            string pythonPackageWheelProperty = null)
+            string pythonPackageWheelProperty = null,
+            string customRequirementsTxtPath = null)
         {
             VirtualEnvironmentName = virtualEnvironmentName;
             VirtualEnvironmentModule = virtualEnvironmentModule;
@@ -36,6 +37,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
             PythonPackageWheelProperty = pythonPackageWheelProperty;
             PythonBuildCommandsFileName = pythonBuildCommandsFileName;
             PythonVersion = pythonVersion;
+            CustomRequirementsTxtPath = customRequirementsTxtPath;
         }
 
         public string VirtualEnvironmentName { get; set; }
@@ -65,5 +67,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
         public string PythonBuildCommandsFileName { get; set; }
 
         public string PythonPackageWheelProperty { get; set; }
+
+        public string CustomRequirementsTxtPath { get; set; }
     }
 }

src/BuildScriptGenerator/Python/PythonPlatform.cs

@@ -563,14 +563,16 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
 
         private void TryLogDependencies(string pythonVersion, ISourceRepo repo)
         {
-            if (!repo.FileExists(PythonConstants.RequirementsFileName))
+            var customRequirementsTxtPath = _pythonScriptGeneratorOptions.CustomRequirementsTxtPath;
+            var requirementsTxtPath = customRequirementsTxtPath == null ? PythonConstants.RequirementsFileName : customRequirementsTxtPath;
+            if (!repo.FileExists(requirementsTxtPath))
             {
                 return;
             }
 
             try
             {
-                var deps = repo.ReadAllLines(PythonConstants.RequirementsFileName)
+                var deps = repo.ReadAllLines(requirementsTxtPath)
                     .Where(line => !line.TrimStart().StartsWith("#"));
                 _logger.LogDependencies(PythonConstants.PlatformName, pythonVersion, deps);
             }

src/BuildScriptGenerator/Python/PythonScriptGeneratorOptions.cs

@@ -12,5 +12,6 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
         public string VirtualEnvironmentName { get; set; }
 
         public string PythonVersion { get; set; }
+        public string CustomRequirementsTxtPath { get; set; }
     }
 }

src/BuildScriptGeneratorCli/Options/BuildScriptGeneratorOptionsSetup.cs

@@ -50,6 +50,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli.Options
             options.AppType = GetStringValue(SettingsKeys.AppType);
             options.BuildCommandsFileName = GetStringValue(SettingsKeys.BuildCommandsFileName);
             options.CompressDestinationDir = GetBooleanValue(SettingsKeys.CompressDestinationDir);
+            options.CustomRequirementsTxtPath = GetStringValue(SettingsKeys.CustomRequirementsTxtPath);
 
             // Dynamic install
             options.EnableDynamicInstall = GetBooleanValue(SettingsKeys.EnableDynamicInstall);

src/BuildScriptGeneratorCli/Options/DetectorOptionsSetup.cs

@@ -21,6 +21,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli.Options
             options.Project = GetStringValue(SettingsKeys.Project);
             options.AppType = GetStringValue(SettingsKeys.AppType);
             options.DisableRecursiveLookUp = GetBooleanValue(SettingsKeys.DisableRecursiveLookUp);
+            options.CustomRequirementsTxtPath = GetStringValue(SettingsKeys.CustomRequirementsTxtPath);
         }
     }
 }

src/BuildScriptGeneratorCli/Options/PythonScriptGeneratorOptionsSetup.cs

@@ -21,6 +21,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli.Options
             options.PythonVersion = GetStringValue(SettingsKeys.PythonVersion);
             options.EnableCollectStatic = !GetBooleanValue(SettingsKeys.DisableCollectStatic);
             options.VirtualEnvironmentName = GetStringValue(SettingsKeys.PythonVirtualEnvironmentName);
+            options.CustomRequirementsTxtPath = GetStringValue(SettingsKeys.CustomRequirementsTxtPath);
         }
     }
 }

src/BuildScriptGeneratorCli/SettingsKeys.cs

@@ -52,5 +52,6 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli
         public const string BuildCommandsFileName = "BUILDCOMMANDS_FILE";
         public const string DynamicInstallRootDir = "DYNAMIC_INSTALL_ROOT_DIR";
         public const string DisableRecursiveLookUp = "DISABLE_RECURSIVE_LOOKUP";
+        public const string CustomRequirementsTxtPath = "CUSTOM_REQUIREMENTSTXT_PATH";
     }
 }

src/Detector/DetectorOptions.cs

@@ -32,5 +32,10 @@ namespace Microsoft.Oryx.Detector
         /// is using. If <c>true</c>, then disable detection for frameworks. Default is <c>false</c>.
         /// </summary>
         public bool DisableFrameworkDetection { get; set; }
+
+        /// <summary>
+        /// Gets or sets the path where a requirements.txt locates.
+        /// </summary>
+        public string CustomRequirementsTxtPath { get; set; }
     }
 }

src/Detector/Python/PythonDetector.cs

@@ -40,15 +40,17 @@ namespace Microsoft.Oryx.Detector.Python
             var appDirectory = string.Empty;
             var hasRequirementsTxtFile = false;
             var hasPyprojectTomlFile = false;
+            var customRequirementsTxtPath = _options.CustomRequirementsTxtPath;
+            var requirementsTxtPath = customRequirementsTxtPath == null ? PythonConstants.RequirementsFileName : customRequirementsTxtPath;
 
-            if (sourceRepo.FileExists(PythonConstants.RequirementsFileName))
+            if (sourceRepo.FileExists(requirementsTxtPath))
             {
-                _logger.LogInformation($"Found {PythonConstants.RequirementsFileName} at the root of the repo.");
+                _logger.LogInformation($"Found {requirementsTxtPath} at the root of the repo.");
                 hasRequirementsTxtFile = true;
 
                 // Warning if missing django module
                 bool hasDjangoModule = false;
-                string filePath = $"{sourceRepo.RootPath}/{PythonConstants.RequirementsFileName}";
+                string filePath = $"{sourceRepo.RootPath}/{requirementsTxtPath}";
                 using (var reader = new StreamReader(filePath))
                 {
                     while (!reader.EndOfStream && !hasDjangoModule)
@@ -62,7 +64,7 @@ namespace Microsoft.Oryx.Detector.Python
                 }
                 if (!hasDjangoModule)
                 {
-                    _logger.LogWarning($"Missing django module in {PythonConstants.RequirementsFileName}");
+                    _logger.LogWarning($"Missing django module in {requirementsTxtPath}");
                 } 
                 else
                 {
@@ -78,7 +80,7 @@ namespace Microsoft.Oryx.Detector.Python
             }
             else
             {
-                string errorMsg = $"Cound not find {PythonConstants.RequirementsFileName} at the root of the repo. More information: https://aka.ms/requirements-not-found";
+                string errorMsg = $"Cound not find {requirementsTxtPath} at the root of the repo. More information: https://aka.ms/requirements-not-found";
                 _logger.LogError(errorMsg);
             }
             if (sourceRepo.FileExists(PythonConstants.PyprojectTomlFileName))

tests/BuildScriptGenerator.Tests/Python/PythonBashBuildSnippetTest.cs

@@ -62,6 +62,34 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
 
         }
 
+        [Fact]
+        public void GeneratedSnippet_ContainsBuildCommand_WhenCustomRequirementsTxtExists()
+        {
+            // Arrange
+            var snippetProps = new PythonBashBuildSnippetProperties(
+                virtualEnvironmentName: null,
+                virtualEnvironmentModule: null,
+                virtualEnvironmentParameters: null,
+                packagesDirectory: "packages_dir",
+                enableCollectStatic: true,
+                compressVirtualEnvCommand: null,
+                compressedVirtualEnvFileName: null,
+                pythonBuildCommandsFileName: FilePaths.BuildCommandsFileName,
+                pythonVersion: "3.6",
+                runPythonPackageCommand: false,
+                customRequirementsTxtPath: "foo/requirements.txt"
+                );
+
+            // Act
+            var text = TemplateHelper.Render(TemplateHelper.TemplateResource.PythonSnippet, snippetProps);
+
+            // Assert
+            Assert.NotEmpty(text);
+            Assert.NotNull(text);
+            Assert.Contains("python -m pip install --cache-dir $PIP_CACHE_DIR --prefer-binary -r $REQUIREMENTS_TXT_FILE", text);
+
+        }
+
         [Fact]
         public void GeneratedSnippet_DoesNotContainCollectStatic_IfDisableCollectStatic_IsTrue()
         {

tests/BuildScriptGenerator.Tests/Python/PythonPlatformTests.cs

@@ -131,6 +131,46 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
             Assert.Null(snippet);
         }
 
+        [Fact]
+        public void GeneratedSnippet_HaveInstallScript_IfCustomRequirementsTxtPathSpecified()
+        {
+            // Arrange
+            var pythonScriptGeneratorOptions = new PythonScriptGeneratorOptions() { CustomRequirementsTxtPath = "foo/requirements.txt" };
+            var commonOptions = new BuildScriptGeneratorOptions() 
+            { 
+                EnableDynamicInstall = true,
+                CustomRequirementsTxtPath = "foo/requirements.txt"
+            };
+            var installerScriptSnippet = "##INSTALLER_SCRIPT##";
+            var versionProvider = new TestPythonVersionProvider(new[] { "3.7.5", "3.8.0" }, defaultVersion: "3.7.5");
+            var platformInstaller = new TestPythonPlatformInstaller(
+                isVersionAlreadyInstalled: false,
+                installerScript: installerScriptSnippet,
+                Options.Create(commonOptions),
+                NullLoggerFactory.Instance);
+            var platform = CreatePlatform(
+                versionProvider,
+                platformInstaller,
+                commonOptions,
+                pythonScriptGeneratorOptions);
+            var repo = new MemorySourceRepo();
+            repo.AddFile("", "foo/requirements.txt");
+            repo.AddFile("print(1)", "bla.py");
+            var context = new BuildScriptGeneratorContext { SourceRepo = repo };
+            var detectorResult = new PythonPlatformDetectorResult
+            {
+                Platform = PythonConstants.PlatformName,
+                PlatformVersion = "3.7.5",
+                HasRequirementsTxtFile = true,
+            };
+
+            // Act
+            var snippet = platform.GetInstallerScriptSnippet(context, detectorResult);
+
+            // Assert
+            Assert.NotNull(snippet);
+        }
+
         [Fact]
         public void GeneratedScript_DoesNotUseVenv()
         {

tests/Detector.Tests/Python/PythonDetectorTest.cs

@@ -219,6 +219,57 @@ namespace Microsoft.Oryx.Detector.Tests.Python
             Assert.Null(result.PlatformVersion);
         }
 
+        [Fact]
+        public void Detect_ReturnsResult_WhenCustomRequirementsFileExists()
+        {
+            // Arrange
+            var options = new DetectorOptions
+            {
+                CustomRequirementsTxtPath = "foo/requirements.txt",
+            };
+            var detector = CreatePythonPlatformDetector(options);
+            var sourceDir = Directory.CreateDirectory(Path.Combine(_tempDirRoot, Guid.NewGuid().ToString("N")))
+                .FullName;
+            var subDirStr = "foo";
+            var subDir = Directory.CreateDirectory(Path.Combine(sourceDir, subDirStr)).FullName;
+            IOHelpers.CreateFile(subDir, "foo==1.1",  "requirements.txt");
+            var repo = new LocalSourceRepo(sourceDir, NullLoggerFactory.Instance);
+            var context = CreateContext(repo);
+
+            // Act
+            var result = detector.Detect(context);
+
+            // Assert
+            var pythonPlatformResult = Assert.IsType<PythonPlatformDetectorResult>(result);
+            Assert.NotNull(pythonPlatformResult);
+            Assert.Equal(PythonConstants.PlatformName, pythonPlatformResult.Platform);
+            Assert.Equal(string.Empty, pythonPlatformResult.AppDirectory);
+            Assert.True(pythonPlatformResult.HasRequirementsTxtFile);
+            Assert.Null(pythonPlatformResult.PlatformVersion);
+        }
+
+        [Fact]
+        public void Detect_ReturnsNull_WhenCustomRequirementsFileDoesNotExist()
+        {
+            // Arrange
+            var options = new DetectorOptions
+            {
+                CustomRequirementsTxtPath = "foo/requirements.txt",
+            };
+            var detector = CreatePythonPlatformDetector(options);
+            var sourceDir = Directory.CreateDirectory(Path.Combine(_tempDirRoot, Guid.NewGuid().ToString("N")))
+                .FullName;
+            IOHelpers.CreateFile(sourceDir, "foo==1.1", "requirements.txt");
+            var repo = new LocalSourceRepo(sourceDir, NullLoggerFactory.Instance);
+            var context = CreateContext(repo);
+
+            // Act
+            var result = detector.Detect(context);
+
+            // Assert
+            Assert.Null(result);
+        }
+
         [Fact]
         public void Detect_ReturnsResult_WhenOnlyJupyterNotebookFilesExist()
         {