Set up CI with 1ES Azure Pipelines (#763)

* Set up CI with 1ES Azure Pipelines

[skip ci]

* Update azure-pipelines-1.yml for Azure Pipelines

* Update azure-pipelines-1.yml for Azure Pipelines

* Fix credscan FP with suppression in test_pkg_config.py

* Update azure-pipelines-1.yml for Azure Pipelines

* Update azure-pipelines-1.yml for Azure Pipelines

* Adding PR and Release pipeline yamls

* Giving pipelines separate names

* Updating comment in release pipeline

* Breaking changing in Bokeh 3.4

.azurepipelines/azure-pipelines-pr.yml

@@ -0,0 +1,42 @@
+# MSTICPy PR pipeline
+
+trigger: none
+name: 1ES-MSTICPy-PR-$(date:yyyyMMdd)$(rev:.r)
+
+resources:
+  repositories:
+  - repository: self
+    type: git
+    ref: main
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+
+extends:
+  template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates
+  parameters:
+    pool:
+      name: MSSecurity-1ES-Build-Agents-Pool
+      image: MSSecurity-1ES-Windows-2022
+      os: windows
+    stages:
+    - stage: buildTasks
+      displayName: BuildTasks
+      jobs:
+      - job: additionalChecks
+        displayName: AdditionalChecks
+        steps:
+        - task: notice@0
+          displayName: NOTICE File Generator
+          # This fails for external forks
+          condition: not(variables['System.PullRequest.IsFork'])
+    sdl:
+      apiScan:
+        enabled: false
+      policheck:
+        enabled: true
+      bandit:
+        enabled: true
+
+

.azurepipelines/azure-pipelines-release.yml

@@ -0,0 +1,41 @@
+# MSTICPy Release pipeline
+
+trigger: none
+name: 1ES-MSTICPy-Rel-$(date:yyyyMMdd)$(rev:.r)
+
+resources:
+  repositories:
+  - repository: self
+    type: git
+    ref: main
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
+  parameters:
+    pool:
+      name: MSSecurity-1ES-Build-Agents-Pool
+      image: MSSecurity-1ES-Windows-2022
+      os: windows
+    stages:
+    - stage: buildTasks
+      displayName: BuildTasks
+      jobs:
+      - job: additionalChecks
+        displayName: AdditionalChecks
+        steps:
+        - task: notice@0
+          displayName: NOTICE File Generator
+          # This fails for external forks
+          condition: not(variables['System.PullRequest.IsFork'])
+    sdl:
+      apiScan:
+        enabled: false
+      policheck:
+        enabled: true
+      bandit:
+        enabled: true
+

azure-pipelines-1.yml

@@ -0,0 +1,45 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger: none
+name: 1ES-MSTICPy-$(date:yyyyMMdd)$(rev:.r)
+
+resources:
+  repositories:
+  - repository: self
+    type: git
+    ref: main
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+
+extends:
+  template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates
+  parameters:
+    pool:
+      name: MSSecurity-1ES-Build-Agents-Pool
+      image: MSSecurity-1ES-Windows-2022
+      os: windows
+    stages:
+    - stage: buildTasks
+      displayName: BuildTasks
+      jobs:
+      - job: additionalChecks
+        displayName: AdditionalChecks
+        steps:
+        - task: notice@0
+          displayName: NOTICE File Generator
+          # This fails for external forks
+          condition: not(variables['System.PullRequest.IsFork'])
+    sdl:
+      apiScan:
+        enabled: false
+      policheck:
+        enabled: true
+      bandit:
+        enabled: true
+
+

conda/conda-reqs.txt

@@ -13,7 +13,7 @@ azure-mgmt-resource>=16.1.0
 azure-storage-blob>=12.5.0
 azure-mgmt-subscription
 beautifulsoup4>=4.0.0
-bokeh>=1.4.0, <4.0.0
+bokeh>=1.4.0, <3.4.0
 cryptography>=3.1
 deprecated>=1.2.4
 dnspython>=2.0.0, <3.0.0

requirements-all.txt

@@ -15,7 +15,7 @@ azure-mgmt-subscription>=3.0.0
 azure-monitor-query>=1.0.0, <=2.0.0
 azure-storage-blob>=12.5.0
 beautifulsoup4>=4.0.0
-bokeh>=1.4.0, <4.0.0
+bokeh>=1.4.0, <3.4.0
 cryptography>=3.1
 deprecated>=1.2.4
 dnspython>=2.0.0, <3.0.0

requirements.txt

@@ -8,7 +8,7 @@ azure-mgmt-keyvault>=2.0.0
 azure-mgmt-subscription>=3.0.0
 azure-monitor-query>=1.0.0, <=2.0.0
 beautifulsoup4>=4.0.0
-bokeh>=1.4.0, <4.0.0
+bokeh>=1.4.0, <3.4.0
 cryptography>=3.1
 deprecated>=1.2.4
 dnspython>=2.0.0, <3.0.0

tests/common/test_pkg_config.py

@@ -16,8 +16,8 @@ import yaml
 from msticpy.common import pkg_config
 from msticpy.context.geoip import GeoLiteLookup, IPStackLookup
 
-from ..unit_test_lib import custom_mp_config, get_test_data_path
 from ..data.queries.test_query_files import validate_queries_file_structure
+from ..unit_test_lib import custom_mp_config, get_test_data_path
 
 _TEST_DATA = get_test_data_path()
 
@@ -112,6 +112,7 @@ def test_geo_ip_settings():
 
         ipstack = IPStackLookup()
         ipstack._check_initialized()
+        # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Test code")]
         check.equal(ipstack._api_key, "987654321-222")