rebase on rc/1.26 branch

.github/workflows/generate-query-help-docs.yml

@@ -8,7 +8,7 @@ on:
      - 'lgtm.com'
   pull_request:
     paths:
-      - 'docs/language/query-help/**'
+      - 'docs/codeql/query-help/**'
 
 jobs:
   build:
@@ -38,10 +38,10 @@ jobs:
       run: unzip -d codeql-cli codeql-linux64.zip
     - name: Set up query help docs folder  
       run: |
-        cp -r codeql/docs/language/query-help/ . ; cp -r codeql/docs/language/global-sphinx-files/ .
+        cp -r codeql/docs/codeql/** .
     - name: Query help to markdown
       run: |
-        PATH="$PATH:codeql-cli/codeql" python codeql/docs/language/query-help-markdown.py
+        PATH="$PATH:codeql-cli/codeql" python codeql/docs/codeql/query-help-markdown.py
     - name: Run Sphinx for query help
       uses: ammaraskar/sphinx-action@master
       with:

docs/codeql/conf.py

@@ -109,5 +109,5 @@ templates_path = ['_templates']
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
-exclude_patterns = ['vale*', '_static', '_templates', 'codeql', 'learn-ql', 'reusables', 'images', 'support', 'ql-training', '_build', '*.py*', 'README.rst']
+exclude_patterns = ['vale*', '_static', '_templates', 'reusables', 'images', 'support', 'ql-training', 'query-help','_build', '*.py*', 'README.rst']
 ##############################################################################

docs/codeql/query-help-markdown.py

@@ -21,8 +21,7 @@ as long as you run the script from one of the following locations:
 # Define which languages and query packs to consider
 languages = [ "cpp", "csharp", "go", "java", "javascript", "python"]
 
-# Query suites to generate help for
-# lgtm-full suites covers all queries used in code scanning and on lgtm.com plus a few more
+# Query suites to generate help for - lgtm suite should cover the queries that users are interested in
 packs = ["lgtm"]
 
 def prefix_repo_nwo(filename):
@@ -110,7 +109,6 @@ except Exception as e:
 # (and assumes the codeql-go repo is in a similar location)
 
 codeql_search_path = "./codeql:./codeql-go"  # will be extended further down
-
 # Extend CodeQL search path by detecting root of the current Git repo (if any). This means that you
 # can run this script from any location within the CodeQL git repository.
 try:
@@ -127,11 +125,18 @@ except:
 # Iterate over all languages and packs, and resolve which queries are part of those packs
 for lang in languages:
 
+    code_scanning_queries = subprocess_run(
+                ["codeql", "resolve", "queries", "--search-path", codeql_search_path, "%s-code-scanning.qls" % (lang)]).stdout.strip()
+    security_extended_queries = subprocess_run(
+                ["codeql", "resolve", "queries", "--search-path", codeql_search_path, "%s-security-extended.qls" % (lang)]).stdout.strip()
+    security_and_quality_queries = subprocess_run(
+                ["codeql", "resolve", "queries", "--search-path", codeql_search_path, "%s-security-and-quality.qls" % (lang)]).stdout.strip()
     # Define empty dictionary to store @name:filename pairs to generate alphabetically sorted Sphinx toctree
     index_file_dictionary = {}
     for pack in packs:
         # Get absolute paths to queries in this pack by using 'codeql resolve queries'
         try:
+            
             queries_subp = subprocess_run(
                 ["codeql", "resolve", "queries", "--search-path", codeql_search_path, "%s-%s.qls" % (lang, pack)])
         except Exception as e:
@@ -189,18 +194,34 @@ for lang in languages:
                     "codeql", "codeql/tree/main").replace(" ", "%20").replace("\\", "/")
             query_link = "[Click to see the query in the CodeQL repository](https://github.com/" + \
                 transform_link + ")\n"
+            
+            if queryfile in code_scanning_queries:
+                cs_suites = lang +'-code-scanning.qls '
+            else:
+                cs_suites = ""
+            if queryfile in security_extended_queries:
+                se_suites = lang + '-security-extended.qls '
+            else:
+                se_suites = ""
+            if queryfile in security_and_quality_queries:
+                sq_suites = lang + '-security-and-quality.qls '
+            else:
+                sq_Suites = ""
+
+            if queryfile in code_scanning_queries or queryfile in security_extended_queries or queryfile in security_and_quality_queries:
+                suites_list = "Query suites: " + cs_suites + se_suites + sq_suites + "\n"
+            else:
+                suites_list = ""
 
             # Join metadata into a literal block and add query link below
             meta_string = "\n"*2 + "```\n" + query_id + query_kind + query_severity + \
-                query_precision + query_tags + "```\n\n" + query_link + "\n"
+                query_precision + query_tags + suites_list + "```\n\n" + query_link + "\n"
 
             # Insert metadata block into query help directly under title
             full_help = query_help.replace("\n", meta_string, 1)
 
-            # Use id property (without language code) to make name for markdown file
-            s = query_id.index("/")
-            # replace "/" with "-"
-            query_name = query_id[s+1:-1].replace("/", "-")
+            # Use id property to make name for markdown file, replacing any "/" characters with "-"
+            query_name = query_id[4:-1].replace("/", "-")
 
             # Populate index_file_dictionary with @name extracted from metadata and corresponding query filename
             index_file_dictionary[query_name_meta] = lang + "/" + query_name

docs/codeql/query-help/conf.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+#
+# CodeQL query help configuration file
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# For details of all possible config values, 
+# see https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project-specific configuration -----------------------------------
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'CodeQL query help'
+
+# Add md parser to process query help markdown files 
+extensions =['recommonmark']
+
+source_suffix = {
+    '.rst': 'restructuredtext',
+    '.md': 'markdown',
+}
+
+# -- Project-specifc options for HTML output ----------------------------------------------
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+html_theme_options = {'font_size': '16px',
+                      'body_text': '#333', 
+                      'link': '#2F1695',
+                      'link_hover': '#2F1695',
+                      'show_powered_by': False,
+                      'nosidebar':True,
+                      'head_font_family': '-apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji"',
+                      }
+
+highlight_language = "none"
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['../_templates']
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['../_static']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+
+exclude_patterns = ['toc-*'] # ignore toc-<lang>.rst files as they are 'included' in index pages

docs/language/query-help/conf.py

@@ -1,74 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# CodeQL query help configuration file
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-# For details of all possible config values, 
-# see https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Project-specific configuration -----------------------------------
-
-import os
-
-# Import global config values
-with open(os.path.abspath("../global-sphinx-files/global-conf.py")) as in_file:
-    exec(in_file.read())
-
-# Set QL as the default language for highlighting code. Set to none to disable 
-# syntax highlighting. If omitted or left blank, it defaults to Python 3. 
-# highlight_language = 'ql'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = u'CodeQL query help'
-
-# Add md parser to process query help markdown files 
-extensions =['recommonmark']
-
-source_suffix = {
-    '.rst': 'restructuredtext',
-    '.md': 'markdown',
-}
-
-# -- Project-specifc options for HTML output ----------------------------------------------
-
-# The version info for this project, if different from version and release in main conf.py file.
-# The short X.Y version.
-# version = u'1.18'
-# The full version, including alpha/beta/rc tags.
-# release = u'1.18'
-
-# -- Currently unused, but potentially useful, configs--------------------------------------
-
-# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
-
-# A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-#html_logo = None
-
-# Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
-
-# Add any extra paths that contain custom files (such as robots.txt or
-# .htaccess) here, relative to this directory. These files are copied
-# directly to the root of the documentation.
-#html_extra_path = []
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-
-exclude_patterns = ['toc-*'] # ignore toc-<lang>.rst files as they are 'included' in index pages