зеркало из https://github.com/mozilla/bedrock.git
Automate Root Store Policy page generation and fix missing indentation of nested lists (#14401)
* Add management command to automatically refresh the root-cert policy doc * First pass at updating the rootstore policy doc with the helper tool * Remove Sentry-notification decorator, as it's irrelevant here
This commit is contained in:
Steve Jalim
GitHub
Родитель
1519e9d9b3
Коммит
e3f73d0dcd
|
|
@ -0,0 +1,164 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
"""Management command to update the root-store cert policy HTML.
|
||||
NOT designed (or needed) to be run on a cron - manual use only."""
|
||||
|
||||
import re
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils.text import slugify
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from markdown_it import MarkdownIt
|
||||
|
||||
SOURCE_CERT_POLICY_DOCUMENT_URL = "https://raw.githubusercontent.com/mozilla/pkipolicy/master/rootstore/policy.md"
|
||||
OUTPUT_FILE_PATH = "bedrock/mozorg/templates/mozorg/about/governance/policies/security/certs/policy.html"
|
||||
WRAPPING_TEMPLATE_PATH = "bedrock/mozorg/templates/mozorg/about/governance/policies/security/certs/_policy_skeleton.html"
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
dest="dry_run",
|
||||
default=False,
|
||||
help="Generate the page and print it, but don't save it.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-q",
|
||||
"--quiet",
|
||||
action="store_true",
|
||||
dest="quiet",
|
||||
default=False,
|
||||
help="If no error occurs, swallow all output.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
action="store",
|
||||
dest="source_url",
|
||||
default=SOURCE_CERT_POLICY_DOCUMENT_URL,
|
||||
help=f"URL to load the policy from. Defaults to {SOURCE_CERT_POLICY_DOCUMENT_URL}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dest",
|
||||
action="store",
|
||||
dest="dest_path",
|
||||
default=OUTPUT_FILE_PATH,
|
||||
help=f"Path to save the generated file to. Defaults to {OUTPUT_FILE_PATH}",
|
||||
)
|
||||
|
||||
def output(self, msg, quiet=None):
|
||||
if not quiet and not self.quiet:
|
||||
print(msg)
|
||||
|
||||
def _wrap_html_with_django_template(self, html):
|
||||
# Note: we don't want to render the actual template at this stage,
|
||||
# we just want to augment what's IN the template with the HTML from
|
||||
# the Policy doc, rendered from Markdown
|
||||
with open(WRAPPING_TEMPLATE_PATH, "r") as fp:
|
||||
template = fp.read()
|
||||
wrapped_html = template.replace("__HTML_POLICY_CONTENT_PLACEHOLDER__", html)
|
||||
return wrapped_html
|
||||
|
||||
def _add_class_to_element(self, tag, klass):
|
||||
tag["class"] = klass
|
||||
return tag
|
||||
|
||||
def _add_header_anchors(self, soup):
|
||||
headings = soup.find_all(re.compile("h[0-9]{1}"))
|
||||
for heading in headings:
|
||||
heading["id"] = slugify(heading.text)
|
||||
return soup
|
||||
|
||||
def _add_toc(self, soup):
|
||||
h2s = soup.find_all("h2")
|
||||
|
||||
toc_list = soup.new_tag("ol")
|
||||
toc_list["class"] = "mzp-u-list-styled"
|
||||
toc_list.append("\n")
|
||||
for h2 in reversed(h2s):
|
||||
new_li = soup.new_tag("li")
|
||||
new_link = soup.new_tag("a", href=f"#{h2['id']}") # new link with anchor
|
||||
new_link.append(h2.text.partition(" ")[-1].strip()) # drop the numbering from the title
|
||||
new_li.append(new_link)
|
||||
toc_list.insert(0, "\n")
|
||||
toc_list.insert(1, " ") # indentation for the li that's coming
|
||||
toc_list.insert(2, new_li)
|
||||
|
||||
# slide it in before the first h2
|
||||
soup.h2.insert_before("\n")
|
||||
soup.h2.insert_before(toc_list)
|
||||
soup.h2.insert_before("\n\n")
|
||||
return soup
|
||||
|
||||
def _tidy_html(self, html):
|
||||
"""
|
||||
1. Add the `class="mzp-c-article-title"` attribute to the `<h1>`
|
||||
(or just keep that line)
|
||||
2. Add `class="mzp-u-list-styled"` to any top-level `<ol>` or `<ul>` elements
|
||||
(no class is required on nested lists)
|
||||
3. Adds anchors to all heading elements
|
||||
4. Add the table of contents as an ordered list above the introduction,
|
||||
with links to each top-level heading.
|
||||
"""
|
||||
|
||||
soup = BeautifulSoup(html, "html5lib")
|
||||
h1 = soup.find("h1")
|
||||
assert h1, "No h1 found!"
|
||||
h1 = self._add_class_to_element(h1, "mzp-c-article-title")
|
||||
|
||||
# We only want the top-level list elements - i.e. the direct children of <body>
|
||||
list_items = soup.find("body").find_all(["ul", "ol"], recursive=False)
|
||||
assert len(list_items) > 0, "No list items found!"
|
||||
for item in list_items:
|
||||
item = self._add_class_to_element(item, "mzp-u-list-styled")
|
||||
|
||||
soup = self._add_header_anchors(soup)
|
||||
|
||||
soup = self._add_toc(soup)
|
||||
|
||||
# We only want the children of the body element, not any fixed up full
|
||||
# <html> node and its children.
|
||||
return "".join([str(x) for x in soup.body.children])
|
||||
|
||||
def handle(self, *args, **options):
|
||||
self.quiet = options["quiet"]
|
||||
source_url = options["source_url"]
|
||||
dest_path = options["dest_path"]
|
||||
dry_run = options["dry_run"]
|
||||
|
||||
self.output(f"Loading Policy doc from {source_url}")
|
||||
|
||||
resp = requests.get(source_url)
|
||||
resp.raise_for_status()
|
||||
text = resp.text
|
||||
|
||||
md = MarkdownIt("commonmark")
|
||||
html = md.render(text)
|
||||
|
||||
self.output("Tidying up the HTML")
|
||||
|
||||
tidied_html = self._tidy_html(html)
|
||||
|
||||
self.output("Wrapping tidied HTML with Django template markup")
|
||||
|
||||
wrapped_html = self._wrap_html_with_django_template(tidied_html)
|
||||
|
||||
if dry_run:
|
||||
self.output("DRY RUN ONLY", quiet=False)
|
||||
self.output(wrapped_html, quiet=False)
|
||||
else:
|
||||
self.output(f"Writing HTML to {dest_path}")
|
||||
with open(
|
||||
dest_path,
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
errors="xmlcharrefreplace",
|
||||
) as output_file:
|
||||
output_file.write(wrapped_html)
|
||||
|
||||
self.output("Done!")
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
{#
|
||||
This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
#}
|
||||
|
||||
{% extends "mozorg/about-base.html" %}
|
||||
|
||||
{% block page_title %}Mozilla Root Store Policy{% endblock %}
|
||||
{% set body_id = "about-policy" %}
|
||||
|
||||
{% block article %}
|
||||
{# Content generated by Markdown processing of
|
||||
# https://raw.githubusercontent.com/mozilla/pkipolicy/master/rootstore/policy.md
|
||||
# Bug 1703063
|
||||
#
|
||||
# Steps to update:
|
||||
# 1. Ensure you have all dev deps installed (automatic if you run `make preflight`)
|
||||
# 2. Run `python manage.py update_root_store_policy_page` (There are options available if needed)
|
||||
# 3. Review changes in your browser at http://localhost:8000/en-US/about/governance/policies/security-group/certs/policy/
|
||||
# 4. Review the git diff locally to ensure class and id attrs are still set
|
||||
# 5. Commit changes
|
||||
#}
|
||||
__HTML_POLICY_CONTENT_PLACEHOLDER__
|
||||
{% endblock %}
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -4,6 +4,7 @@ bpython==0.24
|
|||
braceexpand==0.1.7
|
||||
factory-boy==3.3.0
|
||||
freezegun==1.4.0
|
||||
markdown-it-py>=2.2.0
|
||||
pipdeptree==2.16.1
|
||||
py==1.11.0
|
||||
Pygments>=2.15.0 # to bring it up to a secure version
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ apscheduler==3.10.4 \
|
|||
--hash=sha256:e6df071b27d9be898e486bc7940a7be50b4af2e9da7c08f0744a96d4bd4cef4a \
|
||||
--hash=sha256:fb91e8a768632a4756a585f79ec834e0e27aad5860bac7eaa523d9ccefd87661
|
||||
# via -r requirements/prod.txt
|
||||
asgiref==3.7.2 \
|
||||
--hash=sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e \
|
||||
--hash=sha256:9e0ce3aa93a819ba5b45120216b23878cf6e8525eb3848653452b4192b92afed
|
||||
asgiref==3.8.1 \
|
||||
--hash=sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47 \
|
||||
--hash=sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590
|
||||
# via
|
||||
# -r requirements/prod.txt
|
||||
# django
|
||||
|
|
@ -63,9 +63,9 @@ boto3==1.34.65 \
|
|||
--hash=sha256:b611de58ab28940a36c77d7ef9823427ebf25d5ee8277b802f9979b14e780534 \
|
||||
--hash=sha256:db97f9c29f1806cf9020679be0dd5ffa2aff2670e28e0e2046f98b979be498a4
|
||||
# via -r requirements/prod.txt
|
||||
botocore==1.34.65 \
|
||||
--hash=sha256:399a1b1937f7957f0ee2e0df351462b86d44986b795ced980c11eb768b0e61c5 \
|
||||
--hash=sha256:3b0012d7293880c0a4883883047e93f2888d7317b5e9e8a982a991b90d951f3e
|
||||
botocore==1.34.75 \
|
||||
--hash=sha256:06113ee2587e6160211a6bd797e135efa6aa21b5bde97bf455c02f7dff40203c \
|
||||
--hash=sha256:1d7f683d99eba65076dfb9af3b42fa967c64f11111d9699b65757420902aa002
|
||||
# via
|
||||
# -r requirements/prod.txt
|
||||
# boto3
|
||||
|
|
@ -506,9 +506,9 @@ factory-boy==3.3.0 \
|
|||
--hash=sha256:a2cdbdb63228177aa4f1c52f4b6d83fab2b8623bf602c7dedd7eb83c0f69c04c \
|
||||
--hash=sha256:bc76d97d1a65bbd9842a6d722882098eb549ec8ee1081f9fb2e8ff29f0c300f1
|
||||
# via -r requirements/dev.in
|
||||
faker==24.3.0 \
|
||||
--hash=sha256:5fb5aa9749d09971e04a41281ae3ceda9414f683d4810a694f8a8eebb8f9edec \
|
||||
--hash=sha256:9978025e765ba79f8bf6154c9630a9c2b7f9c9b0f175d4ad5e04b19a82a8d8d6
|
||||
faker==24.4.0 \
|
||||
--hash=sha256:998c29ee7d64429bd59204abffa9ba11f784fb26c7b9df4def78d1a70feb36a7 \
|
||||
--hash=sha256:a5ddccbe97ab691fad6bd8036c31f5697cfaa550e62e000078d1935fa8a7ec2e
|
||||
# via factory-boy
|
||||
fluent-runtime==0.4.0 \
|
||||
--hash=sha256:51fd02582c2363e1106d7051642967a1b7f406dd9c317bd4600a73ede40c5146 \
|
||||
|
|
@ -855,6 +855,10 @@ markdown==3.6 \
|
|||
# -r requirements/prod.txt
|
||||
# django-jinja-markdown
|
||||
# mdx-outline
|
||||
markdown-it-py==3.0.0 \
|
||||
--hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \
|
||||
--hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb
|
||||
# via -r requirements/dev.in
|
||||
markupsafe==2.1.5 \
|
||||
--hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \
|
||||
--hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \
|
||||
|
|
@ -926,6 +930,10 @@ markus[datadog]==4.2.0 \
|
|||
# via
|
||||
# -r requirements/prod.txt
|
||||
# markus
|
||||
mdurl==0.1.2 \
|
||||
--hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \
|
||||
--hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba
|
||||
# via markdown-it-py
|
||||
mdx-outline @ https://github.com/mozmeao/mdx_outline/archive/refs/tags/markdown-3.4-compatibility.tar.gz \
|
||||
--hash=sha256:a78e112f80628246dd45858fe18404aaa8efb8dc81949bb1fbb87e91f9654afa
|
||||
# via -r requirements/prod.txt
|
||||
|
|
@ -1061,9 +1069,9 @@ py==1.11.0 \
|
|||
--hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \
|
||||
--hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378
|
||||
# via -r requirements/dev.in
|
||||
pycparser==2.21 \
|
||||
--hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
|
||||
--hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
|
||||
pycparser==2.22 \
|
||||
--hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \
|
||||
--hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc
|
||||
# via
|
||||
# -r requirements/prod.txt
|
||||
# cffi
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@ apscheduler==3.10.4 \
|
|||
--hash=sha256:e6df071b27d9be898e486bc7940a7be50b4af2e9da7c08f0744a96d4bd4cef4a \
|
||||
--hash=sha256:fb91e8a768632a4756a585f79ec834e0e27aad5860bac7eaa523d9ccefd87661
|
||||
# via -r requirements/prod.in
|
||||
asgiref==3.7.2 \
|
||||
--hash=sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e \
|
||||
--hash=sha256:9e0ce3aa93a819ba5b45120216b23878cf6e8525eb3848653452b4192b92afed
|
||||
asgiref==3.8.1 \
|
||||
--hash=sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47 \
|
||||
--hash=sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590
|
||||
# via
|
||||
# django
|
||||
# django-cors-headers
|
||||
|
|
@ -51,9 +51,9 @@ boto3==1.34.65 \
|
|||
--hash=sha256:b611de58ab28940a36c77d7ef9823427ebf25d5ee8277b802f9979b14e780534 \
|
||||
--hash=sha256:db97f9c29f1806cf9020679be0dd5ffa2aff2670e28e0e2046f98b979be498a4
|
||||
# via -r requirements/prod.in
|
||||
botocore==1.34.65 \
|
||||
--hash=sha256:399a1b1937f7957f0ee2e0df351462b86d44986b795ced980c11eb768b0e61c5 \
|
||||
--hash=sha256:3b0012d7293880c0a4883883047e93f2888d7317b5e9e8a982a991b90d951f3e
|
||||
botocore==1.34.75 \
|
||||
--hash=sha256:06113ee2587e6160211a6bd797e135efa6aa21b5bde97bf455c02f7dff40203c \
|
||||
--hash=sha256:1d7f683d99eba65076dfb9af3b42fa967c64f11111d9699b65757420902aa002
|
||||
# via
|
||||
# boto3
|
||||
# s3transfer
|
||||
|
|
@ -853,9 +853,9 @@ pillow==10.2.0 \
|
|||
--hash=sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48 \
|
||||
--hash=sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868
|
||||
# via -r requirements/prod.in
|
||||
pycparser==2.21 \
|
||||
--hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
|
||||
--hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
|
||||
pycparser==2.22 \
|
||||
--hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \
|
||||
--hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc
|
||||
# via cffi
|
||||
pygithub==2.2.0 \
|
||||
--hash=sha256:41042ea53e4c372219db708c38d2ca1fd4fadab75475bac27d89d339596cfad1 \
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче