Improve handling of preview repo files (#9325)

2024-06-07 16:49:25 -07:00 · 2024-06-07 16:49:25 -07:00 · e894af4615
--- a/toolkit/.gitignore
+++ b/toolkit/.gitignore
@ -1,11 +1,14 @@
-out/
+/out/

 # Temporary toolchain build files that are left behind after a failed build.
-scripts/toolchain/container/.bashrc
-scripts/toolchain/container/coreutils-fix-get-sys_getdents-aarch64.patch
-scripts/toolchain/container/cpio_extern_nocommon.patch
-scripts/toolchain/container/CVE-2021-38185.patch
-scripts/toolchain/container/linker-script-readonly-keyword-support.patch
-scripts/toolchain/container/rpm-define-RPM-LD-FLAGS.patch
-scripts/toolchain/container/texinfo-perl-fix.patch
-scripts/toolchain/container/toolchain-local-wget-list
+/scripts/toolchain/container/.bashrc
+/scripts/toolchain/container/coreutils-fix-get-sys_getdents-aarch64.patch
+/scripts/toolchain/container/cpio_extern_nocommon.patch
+/scripts/toolchain/container/CVE-2021-38185.patch
+/scripts/toolchain/container/linker-script-readonly-keyword-support.patch
+/scripts/toolchain/container/rpm-define-RPM-LD-FLAGS.patch
+/scripts/toolchain/container/texinfo-perl-fix.patch
+/scripts/toolchain/container/toolchain-local-wget-list
+
+# This is generated by the toolkit packaging process, or by a backup generation from the SPECS directory.
+/repos/
--- a/toolkit/Makefile
+++ b/toolkit/Makefile
@ -81,7 +81,6 @@ ENABLE_TRACE                    ?= n
 # Folder defines
 TOOLS_DIR        ?= $(toolkit_root)/tools
 TOOL_BINS_DIR    ?= $(toolkit_root)/out/tools
-PREVIEW_REPO     ?= $(toolkit_root)/repos/azurelinux-official-preview.repo
 RESOURCES_DIR    ?= $(toolkit_root)/resources

 PROJECT_ROOT     ?= $(realpath $(toolkit_root)/..)
@ -164,19 +163,8 @@ ifneq ($(DAILY_BUILD_ID),)
   endif
 else
   ifeq ($(USE_PREVIEW_REPO),y)
-      override PACKAGE_URL_LIST   += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/$(build_arch)
-      override PACKAGE_URL_LIST   += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/debuginfo/$(build_arch)
-      override PACKAGE_URL_LIST   += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/ms-oss/$(build_arch)
-      override SRPM_URL_LIST      += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/srpms
-      ifneq ($(wildcard $(PREVIEW_REPO)),)
-         override REPO_LIST += $(PREVIEW_REPO)
-      else
-         $(warning )
-         $(warning ######################### WARNING #########################)
-         $(warning 'USE_PREVIEW_REPO=y' set but '$(PREVIEW_REPO)' is missing. Regenerate toolkit's 'repos' directory. Remove 'USE_PREVIEW_REPO' for core builds.)
-         $(warning ######################### WARNING #########################)
-         $(warning )
-      endif
+      # Configure the preview repo
+      include $(SCRIPTS_DIR)/preview.mk
   endif
 endif

--- a/toolkit/scripts/imggen.mk
+++ b/toolkit/scripts/imggen.mk
@ -197,7 +197,7 @@ image: $(imager_disk_output_dir) $(imager_disk_output_files) $(go-roast) $(depen
 		$(if $(filter y,$(ENABLE_TRACE)),--enable-trace) \
 		--timestamp-file=$(TIMESTAMP_DIR)/roast.jsonl

-$(image_external_package_cache_summary): $(cached_file) $(go-imagepkgfetcher) $(chroot_worker) $(graph_file) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config)
+$(image_external_package_cache_summary): $(cached_file) $(go-imagepkgfetcher) $(chroot_worker) $(graph_file) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config)
 	$(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=))
 	$(go-imagepkgfetcher) \
 		--input=$(CONFIG_FILE) \
--- a/toolkit/scripts/pkggen.mk
+++ b/toolkit/scripts/pkggen.mk
@ -120,7 +120,7 @@ endif

 # Convert the dependency information in the json file into a graph structure
 # We require all the toolchain RPMs to be available here to help resolve unfixable cyclic dependencies
-$(graph_file): $(specs_file) $(go-grapher) $(toolchain_rpms) $(TOOLCHAIN_MANIFEST) $(pkggen_local_repo) $(graphpkgfetcher_cloned_repo) $(chroot_worker) $(depend_REPO_LIST)
+$(graph_file): $(specs_file) $(go-grapher) $(toolchain_rpms) $(TOOLCHAIN_MANIFEST) $(pkggen_local_repo) $(graphpkgfetcher_cloned_repo) $(chroot_worker) $(depend_REPO_LIST) $(REPO_LIST)
 	$(go-grapher) \
 		--input $(specs_file) \
 		$(logging_command) \
--- a/toolkit/scripts/precache.mk
+++ b/toolkit/scripts/precache.mk
@ -26,7 +26,7 @@ clean-precache:
 # it does, so add the phony target as a dependency to the flag file
 .PHONY: precache_always_run_phony
 pre-cache: $(STATUS_FLAGS_DIR)/precache.flag
-$(STATUS_FLAGS_DIR)/precache.flag: $(go-precacher) $(chroot_worker) $(PRECACHER_SNAPSHOT) precache_always_run_phony
+$(STATUS_FLAGS_DIR)/precache.flag: $(go-precacher) $(chroot_worker) $(PRECACHER_SNAPSHOT) $(depend_REPO_LIST) $(REPO_LIST) precache_always_run_phony
 	@if [ "$(DISABLE_UPSTREAM_REPOS)" = "y" ]; then \
 		echo "ERROR: Upstream repos are disabled (DISABLE_UPSTREAM_REPOS=y), cannot precache RPMs"; \
 		exit 1; \
--- a/toolkit/scripts/preview.mk
+++ b/toolkit/scripts/preview.mk
@ -0,0 +1,42 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Contains:
+#	- Create a preview repo file
+#	- Configuration for getting preview packages
+
+# Ensure USE_PREVIEW_REPO is set to 'y', else this file is not needed.
+ifneq ($(USE_PREVIEW_REPO),y)
+$(error USE_PREVIEW_REPO is not set to 'y' but preview.mk is included)
+endif
+
+######## REPO FILE MANAGEMENT/CREATION ########
+
+preview_repo        := $(toolkit_root)/repos/azurelinux-official-preview.repo
+# Intentionally not using $(SPECS) here to avoid issues with custom SPECS directories
+preview_repo_source := $(PROJECT_ROOT)/SPECS/azurelinux-repos/azurelinux-official-preview.repo
+
+# Stop immediately if the preview repo file is missing and cannot be created.
+ifeq ($(wildcard $(preview_repo_source))$(wildcard $(preview_repo)),)
+	$(warning )
+	$(warning ######################### ERROR #########################)
+	$(warning 'USE_PREVIEW_REPO=y' is set but all of the following failed:)
+	$(warning - packaged default '$(preview_repo)' is missing)
+	$(warning - backup source    '$(preview_repo_source)' is missing)
+	$(warning ######################### ERROR #########################)
+	$(error )
+endif
+
+# Copy the preview repo file to the expected repo location.
+$(preview_repo): $(preview_repo_source)
+	@echo "Regenerating preview repo file from '$<' to '$@'"
+	mkdir -p $(dir $@)
+	cp $< $@
+
+######## VARIABLE CONFIGURATION ########
+
+override PACKAGE_URL_LIST   += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/$(build_arch)
+override PACKAGE_URL_LIST   += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/debuginfo/$(build_arch)
+override PACKAGE_URL_LIST   += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/ms-oss/$(build_arch)
+override SRPM_URL_LIST      += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/srpms
+override REPO_LIST += $(preview_repo)
--- a/toolkit/scripts/repoquerywrapper.mk
+++ b/toolkit/scripts/repoquerywrapper.mk
@ -28,7 +28,7 @@ clean-repo-query:
 #
 .PHONY: repoquerywrapper_always_run_phony
 repo-query: $(STATUS_FLAGS_DIR)/repoquerywrapper.flag
-$(STATUS_FLAGS_DIR)/repoquerywrapper.flag: $(go-repoquerywrapper) $(chroot_worker) repoquerywrapper_always_run_phony 
+$(STATUS_FLAGS_DIR)/repoquerywrapper.flag: $(go-repoquerywrapper) $(chroot_worker) $(depend_REPO_LIST) $(REPO_LIST) repoquerywrapper_always_run_phony
 	@if [ "$(DISABLE_UPSTREAM_REPOS)" = "y" ]; then \
 		echo "ERROR: Upstream repos are disabled (DISABLE_UPSTREAM_REPOS=y), cannot repo-query RPMs"; \
 		exit 1; \