Bug 1731672: Add a Windows Updatebot job r=jewilde,firefox-build-system-reviewers,glandium

This is needed for Angle updates Differential Revision: https://phabricator.services.mozilla.com/D127912
2021-10-25 16:31:09 +00:00 · 2021-10-25 16:31:09 +00:00 · 6c9996e88a
--- a/taskcluster/ci/fetch/kind.yml
+++ b/taskcluster/ci/fetch/kind.yml
@ -17,4 +17,5 @@ jobs-from:
    - resource-monitor.yml
    - toolchain-clang-tidy-external.yml
    - toolchains.yml
+    - updatebot.yml
    - visual-metrics.yml
--- a/taskcluster/ci/fetch/updatebot.yml
+++ b/taskcluster/ci/fetch/updatebot.yml
@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+cloud-sql-proxy-1.25.0:
+    description: Windows executable for Google's Cloud SQL Proxy
+    fetch:
+        type: static-url
+        url: https://storage.googleapis.com/cloudsql-proxy/v1.25.0/cloud_sql_proxy_x64.exe
+        sha256: b922a8d3924499b829bfe53a25382f9b340ea804a738e2a7d8b34957a329a2dd
+        size: 16195072
+        artifact-name: cloud_sql_proxy.exe
+php-windows-7.4.24:
+    description: VC15 x64 Non Thread Safe (2021-Sep-21 16:58:34)
+    fetch:
+        type: static-url
+        url: https://windows.php.net/downloads/releases/php-7.4.24-nts-Win32-vc15-x64.zip
+        sha256: 1e93fb390ab2761a47d27c97225a518759731c632c0dd87436cbf25f19582436
+        size: 26067094
+        artifact-name: php.tar.zst
+        add-prefix: php-win/
+arcanist:
+    description: checkout of arcanist for running on Windows
+    fetch:
+        type: git
+        repo: https://github.com/phacility/arcanist.git
+        revision: a028291f8e5e79b6446ad67ed8b0be9c5ec8f029
--- a/taskcluster/ci/updatebot/kind.yml
+++ b/taskcluster/ci/updatebot/kind.yml
@ -4,6 +4,9 @@
 ---
 loader: gecko_taskgraph.loader.transform:loader

+kind-dependencies:
+    - fetch
+
 transforms:
    - gecko_taskgraph.transforms.release_notifications:transforms
    - gecko_taskgraph.transforms.job:transforms
@ -59,7 +62,7 @@ jobs:
            - hooks:trigger-hook:project-gecko/in-tree-action-1-generic/*
        run:
            using: run-task
-            command: /builds/worker/run.py
+            command: /builds/worker/run.py /builds/worker/checkouts/gecko /builds/worker/updatebot
        notifications:
            subject: 'updatebot cron failed'
            message: 'updatebot cron failed'
@ -67,3 +70,36 @@ jobs:
                - on-failed
                - on-exception
            emails: ["tom@mozilla.com", "jewilde@mozilla.com"]
+
+    cron-win:
+        description: Cron Job for Updatebot (Windows)
+        run-on-projects: []
+        treeherder:
+            kind: other
+            platform: updatebot/all
+            symbol: w-cron
+            tier: 1
+        worker-type: b-win2012
+        worker:
+            max-run-time: 3600
+            taskcluster-proxy: true
+            env:
+                TOOLTOOL_MANIFEST: "taskcluster/ci/updatebot/tooltool-manifests/vc++-2019-runtime.manifest"
+        run:
+            using: run-task
+            cwd: '{checkout}'
+            command: ./taskcluster/docker/updatebot/windows-setup.sh
+            tooltool-downloads: internal
+        scopes:
+            - secrets:get:project/updatebot/{level}/try-sshkey
+            - secrets:get:project/updatebot/{level}/phabricator-token
+            - secrets:get:project/updatebot/{level}/bugzilla-api-key
+            - secrets:get:project/updatebot/{level}/database-password
+            - secrets:get:project/updatebot/{level}/sentry-url
+            - secrets:get:project/updatebot/{level}/sql-proxy-config
+            - hooks:trigger-hook:project-gecko/in-tree-action-1-generic/*
+        fetches:
+            fetch:
+                - cloud-sql-proxy-1.25.0
+                - php-windows-7.4.24
+                - arcanist
--- a/taskcluster/ci/updatebot/tooltool-manifests/vc++-2019-runtime.manifest
+++ b/taskcluster/ci/updatebot/tooltool-manifests/vc++-2019-runtime.manifest
@ -0,0 +1,11 @@
+[
+  {
+    "version": "vcruntime140.dll from Microsoft Visual C++ Redistributable for Visual Studio 2019 ver 14.29.30133.0",
+    "size": 97160,
+    "visibility": "internal",
+    "digest": "aa6afd6bea27f554e3646152d8c4f96f7bcaaa4933f8b7c04346e410f93f23cfa6d29362fd5d51ccbb8b6223e094cd89e351f072ad0517553703f5bf9de28778",
+    "algorithm": "sha512",
+    "filename": "vcruntime140.dll",
+    "unpack": false
+  }
+]
--- a/taskcluster/docker/updatebot/Dockerfile
+++ b/taskcluster/docker/updatebot/Dockerfile
@ -4,6 +4,7 @@ MAINTAINER Tom Ritter <tom@mozilla.com>
 VOLUME /builds/worker/checkouts

 COPY    privileged-setup.sh     /setup/privileged-setup.sh
+COPY    updatebot-version.sh    /setup/updatebot-version.sh
 COPY    setup.sh                /builds/worker/setup.sh
 COPY    run.py                  /builds/worker/run.py
 COPY    hgrc                    /etc/mercurial/hgrc.d/updatebot.rc
--- a/taskcluster/docker/updatebot/arcanist_windows_stream.patch
+++ b/taskcluster/docker/updatebot/arcanist_windows_stream.patch
@ -0,0 +1,19 @@
+diff --git a/src/channel/PhutilSocketChannel.php b/src/channel/PhutilSocketChannel.php
+index 4bd2a47a..2ad3fd85 100644
+--- a/src/channel/PhutilSocketChannel.php
+++ b/src/channel/PhutilSocketChannel.php
+@@ -46,9 +46,11 @@ final class PhutilSocketChannel extends PhutilChannel {
+       if (!$socket) {
+         continue;
+       }
+-      $ok = stream_set_blocking($socket, false);
+-      if (!$ok) {
+-        throw new Exception(pht('Failed to set socket nonblocking!'));
+      if (strtoupper(substr(PHP_OS, 0, 3)) !== 'WIN') {
+        $ok = stream_set_blocking($socket, false);
+        if (!$ok) {
+          throw new Exception(pht('Failed to set socket nonblocking!'));
+        }
+       }
+     }
+ 
--- a/taskcluster/docker/updatebot/hgrc
+++ b/taskcluster/docker/updatebot/hgrc
@ -1,5 +1,7 @@
 [ui]
-ssh = ssh -i /builds/worker/updatebot/id_rsa -l updatebot@mozilla.com
+ssh = ssh -i $HOME/id_rsa -l updatebot@mozilla.com -o UserKnownHostsFile=$HOME/ssh_known_hosts
 username = Updatebot <updatebot@mozilla.com>
+
 [extensions]
 strip =
+push-to-try = $HOME/.mozbuild/version-control-tools/hgext/push-to-try
--- a/taskcluster/docker/updatebot/privileged-setup.sh
+++ b/taskcluster/docker/updatebot/privileged-setup.sh
@ -5,7 +5,9 @@

 set -vex

-export UPDATEBOT_REVISION=d3d7a79ac23a1d2fac8c39a55f4e8818e6e51426
+. ./updatebot-version.sh # Get UPDATEBOT_REVISION
+
+# If you edit this, be sure to edit fetch/updatebot.yml
 export SQLPROXY_REVISION=fb1939ab92846761595833361c6b0b0ecd543861

 export DEBIAN_FRONTEND=noninteractive
@ -51,7 +53,7 @@ go get github.com/GoogleCloudPlatform/cloudsql-proxy/cmd/cloud_sql_proxy@$SQLPRO
 cd /builds/worker/
 git clone https://github.com/mozilla-services/updatebot.git
 cd updatebot
-git checkout $UPDATEBOT_REVISION
+git checkout "$UPDATEBOT_REVISION"

 # Set up dependencies
 cd /builds/worker/
--- a/taskcluster/docker/updatebot/run.py
+++ b/taskcluster/docker/updatebot/run.py
@ -14,11 +14,27 @@ import os
 import stat
 import base64
 import signal
+import platform
 import requests
 import subprocess
 import taskcluster

-# Bump this number when you need to cause a commit for the job to re-run: 17
+# Bump this number when you need to cause a commit for the job to re-run: 20
+
+if len(sys.argv) < 3:
+    print("Usage:", sys.argv[0], "gecko-dev-path updatebot-path [moz-fetches-dir]")
+    sys.exit(1)
+
+GECKO_DEV_PATH = sys.argv[1].replace("/", os.path.sep)
+UPDATEBOT_PATH = sys.argv[2].replace("/", os.path.sep)
+
+# Only needed on Windows
+if len(sys.argv) > 3:
+    FETCHES_PATH = sys.argv[3].replace("/", os.path.sep)
+else:
+    FETCHES_PATH = None
+
+HOME_PATH = os.path.expanduser("~")

 OPERATING_MODE = (
    "prod"
@ -27,7 +43,6 @@ OPERATING_MODE = (
    else "dev"
 )

-GECKO_DEV_PATH = "/builds/worker/checkouts/gecko"
 DEV_PHAB_URL = "https://phabricator-dev.allizom.org/"
 PROD_PHAB_URL = "https://phabricator.services.mozilla.com/"

@ -68,8 +83,6 @@ database_config = get_secret("database-password")
 sentry_url = get_secret("sentry-url")
 sql_proxy_config = get_secret("sql-proxy-config")

-os.chdir("/builds/worker/updatebot")
-
 # Update Updatebot =======================================
 if OPERATING_MODE == "dev":
    """
@ -86,19 +99,33 @@ if OPERATING_MODE == "dev":

    Therefore, we only do this in dev mode when running on try.
    """
+
+    os.chdir(UPDATEBOT_PATH)
    log("Performing git repo update...")
-    r = subprocess.run(["git", "symbolic-ref", "-q", "HEAD"])
+    command = ["git", "symbolic-ref", "-q", "HEAD"]
+
+    r = subprocess.run(command)
    if r.returncode == 0:
        # This indicates we are on a branch, and not a specific revision
        subprocess.check_call(["git", "pull", "origin"])

 # Set Up SSH & Phabricator ==============================
+os.chdir(HOME_PATH)
 log("Setting up ssh and phab keys...")
 with open("id_rsa", "w") as sshkey:
    sshkey.write(try_sshkey)
 os.chmod("id_rsa", stat.S_IRUSR | stat.S_IWUSR)

-arcrc = open("/builds/worker/.arcrc", "w")
+arc_filename = ".arcrc"
+if platform.system() == "Windows":
+    arc_path = os.path.join(FETCHES_PATH, "..", "AppData", "Roaming")
+    os.makedirs(arc_path, exist_ok=True)
+    os.chdir(arc_path)
+    log("Writing %s to %s" % (arc_filename, arc_path))
+else:
+    os.chdir(HOME_PATH)
+
+arcrc = open(arc_filename, "w")
 towrite = """
 {
  "hosts": {
@ -114,41 +141,53 @@ towrite = """
 )
 arcrc.write(towrite)
 arcrc.close()
-os.chmod("/builds/worker/.arcrc", stat.S_IRUSR | stat.S_IWUSR)
+os.chmod(arc_filename, stat.S_IRUSR | stat.S_IWUSR)

 # Set up the Cloud SQL Proxy =============================
+os.chdir(HOME_PATH)
 log("Setting up cloud_sql_proxy...")
-os.chdir("/builds/worker/")
 with open("sql-proxy-key", "w") as proxy_key_file:
    proxy_key_file.write(
        base64.b64decode(sql_proxy_config["key-value"]).decode("utf-8")
    )

 instance_name = sql_proxy_config["instance-name"]
-sql_proxy_command = (
-    "./go/bin/cloud_sql_proxy -instances="
-    + instance_name
-    + "=tcp:3306 -credential_file=sql-proxy-key"
-)
-sql_proxy = subprocess.Popen(
-    sql_proxy_command,
-    stdout=subprocess.PIPE,
-    stderr=subprocess.PIPE,
-    shell=True,
-    start_new_session=True,
+if platform.system() == "Linux":
+    sql_proxy_command = "/builds/worker/go/bin/cloud_sql_proxy"
+else:
+    sql_proxy_command = os.path.join(UPDATEBOT_PATH, "..", "cloud_sql_proxy.exe")
+
+sql_proxy_command += (
+    " -instances=" + instance_name + "=tcp:3306 -credential_file=sql-proxy-key"
 )
+sql_proxy_args = {
+    "stdout": subprocess.PIPE,
+    "stderr": subprocess.PIPE,
+    "shell": True,
+    "start_new_session": True,
+}
+
+if platform.system() == "Windows":
+    si = subprocess.STARTUPINFO()
+    si.dwFlags = subprocess.CREATE_NEW_PROCESS_GROUP
+
+    sql_proxy_args["startupinfo"] = si
+
+sql_proxy = subprocess.Popen((sql_proxy_command), **sql_proxy_args)
+
 try:
    (stdout, stderr) = sql_proxy.communicate(input=None, timeout=2)
    log("sql proxy stdout:", stdout.decode("utf-8"))
    log("sql proxy stderr:", stderr.decode("utf-8"))
 except subprocess.TimeoutExpired:
    log("no sqlproxy output in 2 seconds, this means it probably didn't error.")
+    log("sqlproxy pid:", sql_proxy.pid)

 database_config["host"] = "127.0.0.1"

 # Vendor =================================================
 log("Getting Updatebot ready...")
-os.chdir("/builds/worker/updatebot")
+os.chdir(UPDATEBOT_PATH)
 localconfig = {
    "General": {
        "env": OPERATING_MODE,
@ -175,8 +214,10 @@ config.write("localconfig = " + str(localconfig))
 config.close()

 log("Running updatebot")
-subprocess.check_call(["poetry", "run", "./automation.py"])
+# On Windows, Updatebot is run by windows-setup.sh
+if platform.system() == "Linux":
+    subprocess.check_call(["python3", "-m", "poetry", "run", "./automation.py"])

-# Clean up ===============================================
-log("Killing cloud_sql_proxy")
-os.killpg(sql_proxy.pid, signal.SIGTERM)
+    # Clean up ===============================================
+    log("Killing cloud_sql_proxy")
+    os.kill(sql_proxy.pid, signal.SIGTERM)
--- a/taskcluster/docker/updatebot/setup.sh
+++ b/taskcluster/docker/updatebot/setup.sh
@ -5,7 +5,9 @@

 set -vex

-whoami
+# Copy the system known_hosts to the home directory so we have uniformity with Windows
+# and the ssh command will find them in the same place.
+cp /etc/ssh/ssh_known_hosts $HOME/ssh_known_hosts

 # If poetry is not run as worker, then it won't work when run as user later.
 cd /builds/worker/updatebot
--- a/taskcluster/docker/updatebot/updatebot-version.sh
+++ b/taskcluster/docker/updatebot/updatebot-version.sh
@ -0,0 +1 @@
+export UPDATEBOT_REVISION=d3d7a79ac23a1d2fac8c39a55f4e8818e6e51426
--- a/taskcluster/docker/updatebot/windows-php.ini
+++ b/taskcluster/docker/updatebot/windows-php.ini
@ -0,0 +1,130 @@
+[PHP]
+
+engine = On
+short_open_tag = Off
+precision = 14
+output_buffering = 4096
+zlib.output_compression = Off
+implicit_flush = Off
+unserialize_callback_func =
+serialize_precision = -1
+disable_functions =
+disable_classes =
+zend.enable_gc = On
+zend.exception_ignore_args = Off
+expose_php = On
+max_execution_time = 30
+max_input_time = 60
+memory_limit = 128M
+error_reporting = E_ALL
+display_errors = On
+display_startup_errors = On
+log_errors = On
+log_errors_max_len = 1024
+ignore_repeated_errors = Off
+ignore_repeated_source = Off
+report_memleaks = On
+variables_order = "GPCS"
+request_order = "GP"
+register_argc_argv = Off
+auto_globals_jit = On
+post_max_size = 8M
+auto_prepend_file =
+auto_append_file =
+default_mimetype = "text/html"
+default_charset = "UTF-8"
+doc_root =
+user_dir =
+extension_dir = "ext"
+enable_dl = Off
+file_uploads = On
+upload_max_filesize = 2M
+max_file_uploads = 20
+allow_url_fopen = On
+allow_url_include = Off
+default_socket_timeout = 60
+extension=curl
+
+[CLI Server]
+cli_server.color = On
+
+[Pdo_mysql]
+pdo_mysql.default_socket=
+
+[mail function]
+SMTP = localhost
+smtp_port = 25
+mail.add_x_header = Off
+
+[ODBC]
+odbc.allow_persistent = On
+odbc.check_persistent = On
+odbc.max_persistent = -1
+odbc.max_links = -1
+odbc.defaultlrl = 4096
+odbc.defaultbinmode = 1
+
+[MySQLi]
+mysqli.max_persistent = -1
+mysqli.allow_persistent = On
+mysqli.max_links = -1
+mysqli.default_port = 3306
+mysqli.default_socket =
+mysqli.default_host =
+mysqli.default_user =
+mysqli.default_pw =
+mysqli.reconnect = Off
+
+[mysqlnd]
+mysqlnd.collect_statistics = On
+mysqlnd.collect_memory_statistics = On
+
+[PostgreSQL]
+pgsql.allow_persistent = On
+pgsql.auto_reset_persistent = Off
+pgsql.max_persistent = -1
+pgsql.max_links = -1
+pgsql.ignore_notice = 0
+pgsql.log_notice = 0
+
+[bcmath]
+bcmath.scale = 0
+
+[Session]
+session.save_handler = files
+session.use_strict_mode = 0
+session.use_cookies = 1
+session.use_only_cookies = 1
+session.name = PHPSESSID
+session.auto_start = 0
+session.cookie_lifetime = 0
+session.cookie_path = /
+session.cookie_domain =
+session.cookie_httponly =
+session.cookie_samesite =
+session.serialize_handler = php
+session.gc_probability = 1
+session.gc_divisor = 1000
+session.gc_maxlifetime = 1440
+session.referer_check =
+session.cache_limiter = nocache
+session.cache_expire = 180
+session.use_trans_sid = 0
+session.sid_length = 26
+session.trans_sid_tags = "a=href,area=href,frame=src,form="
+session.sid_bits_per_character = 5
+
+[Assertion]
+zend.assertions = 1
+
+[Tidy]
+tidy.clean_output = Off
+
+[soap]
+soap.wsdl_cache_enabled=1
+soap.wsdl_cache_dir="/tmp"
+soap.wsdl_cache_ttl=86400
+soap.wsdl_cache_limit = 5
+
+[ldap]
+ldap.max_links = -1
--- a/taskcluster/docker/updatebot/windows-setup.sh
+++ b/taskcluster/docker/updatebot/windows-setup.sh
@ -0,0 +1,84 @@
+#!/bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+set -vex
+
+. ./taskcluster/docker/updatebot/updatebot-version.sh # Get UPDATEBOT_REVISION
+
+export HOME=$(python3 -c "import os;print(os.path.expanduser('~'))")
+GECKO_PATH="$PWD"
+UPDATEBOT_PATH="$MOZ_FETCHES_DIR/updatebot"
+
+# MOZ_FETCHES_DIR is in Z:/ format.  When we update the PATH we need to use
+#   /z/ format.  Fortunately, we can translate them like so:
+cd $MOZ_FETCHES_DIR
+MOZ_FETCHES_PATH="$PWD"
+
+#########################################################
+# Install dependencies
+
+# Git is at /c/Program Files/Git/cmd/git.exe
+#  It's in PATH for this script (confusingly) but not in PATH so we need to add it
+which git
+export PATH="/c/Program Files/Git/cmd:$PATH"
+
+# php & arcanist
+if [ -n "$TOOLTOOL_MANIFEST" ]; then
+ . "$GECKO_PATH/taskcluster/scripts/misc/tooltool-download.sh"
+fi
+
+cp "$MOZ_FETCHES_DIR/vcruntime140.dll" "$MOZ_FETCHES_DIR/php-win"
+cp "$GECKO_PATH/taskcluster/docker/updatebot/windows-php.ini" "$MOZ_FETCHES_DIR/php-win/php.ini"
+
+cd "$MOZ_FETCHES_DIR/arcanist.git"
+patch -p1 < "$GECKO_PATH/taskcluster/docker/updatebot/arcanist_windows_stream.patch"
+cd "$MOZ_FETCHES_DIR"
+
+export PATH="$MOZ_FETCHES_PATH/php-win:$PATH"
+export PATH="$MOZ_FETCHES_PATH/arcanist.git/bin:$PATH"
+
+
+
+ls $MOZ_FETCHES_PATH/arcanist.git/bin
+
+# get Updatebot
+cd "$MOZ_FETCHES_DIR"
+git clone https://github.com/mozilla-services/updatebot.git
+cd updatebot
+git checkout "$UPDATEBOT_REVISION"
+
+# base python needs
+python3 -m pip install --no-warn-script-location --user -U pip
+python3 -m pip install --no-warn-script-location --user poetry wheel requests setuptools
+
+# updatebot dependencies
+cd "$UPDATEBOT_PATH"
+python3 -m poetry install
+
+# taskcluster secrets and writing out localconfig
+cd "$GECKO_PATH"
+python3 ./taskcluster/docker/updatebot/run.py "$GECKO_PATH" "$UPDATEBOT_PATH" "$MOZ_FETCHES_PATH"
+
+# mercurial configuration
+cp "$GECKO_PATH/taskcluster/docker/updatebot/hgrc" "$HOME/.hgrc"
+# Windows is not happy with $HOME in the hgrc so we need to do a hack to replace it
+#   with the actual value
+( echo "cat <<EOF" ; cat "$HOME/.hgrc" ) | sh > tmp
+mv tmp "$HOME/.hgrc"
+
+# ssh known hosts
+cp "$GECKO_PATH/taskcluster/docker/push-to-try/known_hosts" "$HOME/ssh_known_hosts"
+
+#########################################################
+# Run it
+export PYTHONIOENCODING=utf8
+export PYTHONUNBUFFERED=1
+
+cd "$UPDATEBOT_PATH"
+python3 -m poetry run python3 ./automation.py
+
+#########################################################
+echo "Killing SQL Proxy"
+taskkill -f -im cloud_sql_proxy.exe || true
				`@ -0,0 +1 @@`
				`export UPDATEBOT_REVISION=d3d7a79ac23a1d2fac8c39a55f4e8818e6e51426`