From 77170a1639364c140bd68b32dd0fd841aa40c65e Mon Sep 17 00:00:00 2001 From: jhugman Date: Wed, 20 Sep 2023 21:48:12 +0100 Subject: [PATCH] Add libnimbus_experimenter.so to experimenter (#9455) Because - we haven't yet cross compiled libnimbus_experimenter.so for developer machines This commit - Adds an extra git checkout / cargo build to get libnimbus_experimener.so into experimenter. N.B. this uses the `--library` form of `uniffi_bindgen`. Once lands, we should remove the `cdylib_name` entries in the FML and Nimbus `uniffi.toml` files in application-services. --- Makefile | 2 +- cirrus/server/Dockerfile | 11 ++---- experimenter/Dockerfile | 38 ++++++++++++++++++- .../features/manifests/nimbus_fml_loader.py | 12 +++--- .../features/tests/test_nimbus_fml_loader.py | 2 +- .../nimbus/utils/Dockerfile-rust-image | 17 ++++++--- 6 files changed, 58 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 1b3ef889a..ef7ff7d1e 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ LOAD_LOCALES = python manage.py loaddata ./experimenter/base/fixtures/locales.js LOAD_LANGUAGES = python manage.py loaddata ./experimenter/base/fixtures/languages.json LOAD_FEATURES = python manage.py load_feature_configs LOAD_DUMMY_EXPERIMENTS = [[ -z $$SKIP_DUMMY ]] && python manage.py load_dummy_experiments || python manage.py load_dummy_projects -PYTHON_PATH_SDK = PYTHONPATH=/application-services/components/nimbus/src +PYTHON_PATH_SDK = PYTHONPATH=/application-services/megazord JETSTREAM_CONFIG_URL = https://github.com/mozilla/metric-hub/archive/main.zip diff --git a/cirrus/server/Dockerfile b/cirrus/server/Dockerfile index ba8cacb00..ae15c244f 100644 --- a/cirrus/server/Dockerfile +++ b/cirrus/server/Dockerfile @@ -3,8 +3,6 @@ FROM rust:buster AS rust-builder ARG as_version -RUN rustup target add x86_64-unknown-linux-musl -RUN apt update && apt install -y musl-tools musl-dev RUN update-ca-certificates RUN apt-get update -qqy \ @@ -28,8 +26,7 @@ RUN git submodule init RUN git submodule update --recursive RUN cargo build --manifest-path megazords/cirrus/Cargo.toml --release -RUN cargo uniffi-bindgen generate components/nimbus/src/cirrus.udl --language python -RUN cargo uniffi-bindgen generate components/support/nimbus-fml/src/fml.udl --language python +RUN cargo uniffi-bindgen generate --library target/release/libcirrus.so --language python --out-dir . # Second stage: Build Python environment with dependencies FROM python:3.11.2-slim-buster as python-builder @@ -37,7 +34,7 @@ FROM python:3.11.2-slim-buster as python-builder # Set working directory for the container WORKDIR /cirrus -# Install curl and bash utilities +# Install curl and bash utilities to build Glean RUN apt-get update && \ apt-get -y install curl bash build-essential && \ rm -rf /var/lib/apt/lists/* @@ -65,9 +62,9 @@ FROM python:3.11.2-slim-buster as deploy WORKDIR /cirrus # Copy specific Rust components -COPY --from=rust-builder /application-services/components/nimbus/src/cirrus.py /application-services/cirrus_sdk.py COPY --from=rust-builder /application-services/target/release/libcirrus.so /application-services/libcirrus.so -COPY --from=rust-builder /application-services/components/support/nimbus-fml/src/fml.py /application-services/fml_sdk.py +COPY --from=rust-builder /application-services/cirrus.py /application-services/cirrus_sdk.py +COPY --from=rust-builder /application-services/fml.py /application-services/fml_sdk.py # Copy Python site packages and scripts COPY --from=python-builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages diff --git a/experimenter/Dockerfile b/experimenter/Dockerfile index 6e2e8addf..299a60dd5 100644 --- a/experimenter/Dockerfile +++ b/experimenter/Dockerfile @@ -42,7 +42,41 @@ RUN apt-get update RUN apt-get --no-install-recommends install -y apt-utils ca-certificates postgresql-client yarn parallel +# Rust packages +# This is temporary while megazord packaging is pending. +WORKDIR /application-services +ARG as_version + +RUN git clone https://github.com/mozilla/application-services.git . +RUN if [[ -z "${as_version}" ]] ; then \ + echo "No application-services version supplied, using latest commit." ; \ + else \ + git fetch ; \ + fi + +RUN git submodule init +RUN git submodule update --recursive + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | /bin/sh -s -- -y --profile minimal +ENV PATH "/root/.cargo/bin:$PATH" +RUN mkdir -p /application-services/megazord + +RUN cargo build --manifest-path /application-services/megazords/nimbus-experimenter/Cargo.toml --release +RUN cargo uniffi-bindgen generate --library /application-services/target/release/libnimbus_experimenter.so --language python --out-dir /application-services/megazord + +RUN mv /application-services/megazord/fml.py /application-services/megazord/rust_fml.py +RUN mv /application-services/megazord/nimbus.py /application-services/megazord/rust_sdk.py +RUN mv /application-services/megazord/errorsupport.py /application-services/megazord/rust_errorsupport.py +RUN mv /application-services/megazord/remote_settings.py /application-services/megazord/rust_remote_settings.py + +# The uniffi.toml file setting cdylib_name is set to "cirrus", which means the generated python is expecting +# the library to be called libcirrus.so. +RUN cp /application-services/target/release/libnimbus_experimenter.so /application-services/megazord/libcirrus.so +# RUN cp /application-services/target/release/libnimbus_experimenter.so /application-services/megazord/libnimbus_experimenter.so +ENV PYTHONPATH=$PYTHONPATH:/application-services/megazord + # Python packages +WORKDIR /experimenter RUN curl -sSL https://install.python-poetry.org | python3 - ENV PATH "/root/.local/bin:$PATH" RUN poetry config virtualenvs.create false @@ -84,8 +118,8 @@ COPY --from=file-loader /experimenter/experimenter/legacy/legacy-ui/ /experiment RUN yarn workspace @experimenter/core build COPY --from=file-loader /experimenter/experimenter/nimbus-ui/ /experimenter/experimenter/nimbus-ui/ RUN yarn workspace @experimenter/nimbus-ui build -COPY --from=file-loader /experimenter/experimenter/theme/ /experimenter/experimenter/theme/ -COPY --from=file-loader /experimenter/experimenter/templates/ /experimenter/experimenter/templates/ +COPY --from=file-loader /experimenter/experimenter/theme/ /experimenter/experimenter/theme/ +COPY --from=file-loader /experimenter/experimenter/templates/ /experimenter/experimenter/templates/ RUN yarn --cwd /experimenter/experimenter/theme/static_src build # Deploy image diff --git a/experimenter/experimenter/features/manifests/nimbus_fml_loader.py b/experimenter/experimenter/features/manifests/nimbus_fml_loader.py index c444660a4..67f1aa531 100644 --- a/experimenter/experimenter/features/manifests/nimbus_fml_loader.py +++ b/experimenter/experimenter/features/manifests/nimbus_fml_loader.py @@ -2,12 +2,10 @@ import logging import os import yaml +from rust_fml import FmlClient from experimenter.settings import BASE_DIR -# Todo: Connect FML https://mozilla-hub.atlassian.net/browse/EXP-3791 -# from nimbus-experimenter import Fml - logger = logging.getLogger() @@ -34,7 +32,7 @@ class NimbusFmlLoader: else: logger.error("Failed to find fml path for application: " + application) - def create(self, path: str, channel: str): - return "success" - # Todo: Connect FML https://mozilla-hub.atlassian.net/browse/EXP-3791 - # return Fml.new(path, channel) + def create(self, path: str, channel: str) -> FmlClient: + return FmlClient.new_with_ref( + "@mozilla-mobile/firefox-android/fenix/app/nimbus.fml.yaml", "release", "main" + ) diff --git a/experimenter/experimenter/features/tests/test_nimbus_fml_loader.py b/experimenter/experimenter/features/tests/test_nimbus_fml_loader.py index 80457c942..8eb77b1dc 100644 --- a/experimenter/experimenter/features/tests/test_nimbus_fml_loader.py +++ b/experimenter/experimenter/features/tests/test_nimbus_fml_loader.py @@ -43,4 +43,4 @@ class TestNimbusFmlLoader(TestCase): loader = NimbusFmlLoader(application, channel) response = loader.create("my/favorite/path", "ziggy") # Todo: Connect FML https://mozilla-hub.atlassian.net/browse/EXP-3791 - self.assertEqual(response, "success") + self.assertIsNotNone(response) diff --git a/experimenter/tests/integration/nimbus/utils/Dockerfile-rust-image b/experimenter/tests/integration/nimbus/utils/Dockerfile-rust-image index 5632e17a7..3ea5cd4c3 100644 --- a/experimenter/tests/integration/nimbus/utils/Dockerfile-rust-image +++ b/experimenter/tests/integration/nimbus/utils/Dockerfile-rust-image @@ -32,20 +32,25 @@ RUN git submodule init RUN git submodule update --recursive RUN ./libs/verify-desktop-environment.sh +RUN mkdir -p /application-services/megazord RUN cargo build -RUN cargo build --manifest-path megazords/full/Cargo.toml --release -RUN cargo uniffi-bindgen generate components/nimbus/src/nimbus.udl --language python +RUN cargo build --manifest-path megazords/nimbus-experimenter/Cargo.toml --release +RUN cargo uniffi-bindgen generate --library target/release/libnimbus_experimenter.so --language python --out-dir /application-services/megazord + +RUN mv megazord/nimbus.py megazord/nimbus_rust.py +# This should be called libnimbus_experimenter.so; once all uniffi-bindgen calls in this repo have moved to --library, we +# can move this libnimbus_experimenter.so +RUN mv target/release/libnimbus_experimenter.so megazord/libcirrus.so -RUN mv components/nimbus/src/nimbus.py components/nimbus/src/nimbus_rust.py -RUN mv target/release/libmegazord.so components/nimbus/src/libuniffi_nimbus.so FROM python:3.11-buster -COPY --from=builder /application-services /application-services +COPY --from=builder /application-services/megazord /application-services/megazord +ENV PYTHON_PATH=$PYTHON_PATH:/application-services/megazord RUN apt-get update -qqy \ && apt-get -qqy install \ tox -WORKDIR /code \ No newline at end of file +WORKDIR /code