Merge branch 'master' of https://github.com/HarelKatz/mstic into HarelKatz-master

.github/workflows/piavpn-feed.yml

@@ -0,0 +1,44 @@
+name: Daily PIA VPN Servers Feed
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *" # Runs at 00:00 AM (UTC) everyday (Check https://crontab.guru/#0_0_*_*_*)
+
+jobs:
+  download-piavpnservers:
+    name: Download PIA VPN Servers via API
+    runs-on: ubuntu-latest
+    steps:
+      - name: 🍽️ Checkout the repo
+        uses: actions/checkout@v3
+        with:
+          path: master
+          fetch-depth: 0
+
+      - name: 🐍 Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.8"
+          architecture: "x64"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests pandas
+
+      - name: 🚀 Run automation script
+        run: python  master/.script/get-piavpnservers.py
+
+      - name: Commit files
+        run: |
+          cd master
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add *
+          git commit -m "Adding new PIA vpn servers daily feed" -a
+
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          directory: "master"

.script/get-piavpnservers.py

@@ -0,0 +1,113 @@
+import requests
+import json
+import sys
+import logging
+import pandas as pd
+from pathlib import Path
+
+
+def download_pia_vpn_servers(output_file):
+
+    new_data = []
+
+
+    # V4
+    logging.info(f"Downloading PIA VPN server list using API from https://serverlist.piaservers.net/vpninfo/servers/v4")
+
+
+    r_v4 = requests.get("https://serverlist.piaservers.net/vpninfo/servers/v4")
+
+
+    my_json_v4 = r_v4.content.decode("utf8")
+
+    # Remove everything after the last brace
+    my_json_v4 = my_json_v4[:my_json_v4.rfind('}')+1]
+
+
+    data_v4 = json.loads(my_json_v4)["regions"]
+
+    for region in data_v4:
+        for protocol_type in region["servers"]:
+            new_data.append({"ip": region["servers"][protocol_type][0]["ip"], "cn": region["servers"][protocol_type][0]["cn"], "id": region["id"],
+                             "name": region["name"], "country": region["country"], "dns": region["dns"], "port_forward": str(region["port_forward"]), "protocol": protocol_type, "offline": ""})
+            
+    
+    "----------------------------------------------------------------------------"
+
+
+    # V5
+    logging.info(f"Downloading PIA VPN server list using API from https://serverlist.piaservers.net/vpninfo/servers/v5")
+
+
+    r_v5 = requests.get("https://serverlist.piaservers.net/vpninfo/servers/v5")
+
+
+    my_json_v5 = r_v5.content.decode("utf8")
+
+    # Remove everything after the last brace
+    my_json_v5 = my_json_v5[:my_json_v5.rfind('}')+1]
+
+
+    data_v5 = json.loads(my_json_v5)["regions"]
+
+    for region in data_v5:
+        for protocol_type in region["servers"]:
+            new_data.append({"ip": region["servers"][protocol_type][0]["ip"], "cn": region["servers"][protocol_type][0]["cn"], "id": region["id"],
+                             "name": region["name"], "country": region["country"], "dns": region["dns"], "port_forward": str(region["port_forward"]), "protocol": protocol_type, "offline": region["offline"] })
+            
+    
+    "----------------------------------------------------------------------------"
+
+
+    # V6
+    logging.info(f"Downloading PIA VPN server list using API from https://serverlist.piaservers.net/vpninfo/servers/v6")
+
+
+    r_v6 = requests.get("https://serverlist.piaservers.net/vpninfo/servers/v6")
+
+
+    my_json_v6 = r_v6.content.decode("utf8")
+
+    # Remove everything after the last brace
+    my_json_v6 = my_json_v6[:my_json_v6.rfind('}')+1]
+
+    data_v6 = json.loads(my_json_v6)["regions"]
+
+    for region in data_v6:
+        for protocol_type in region["servers"]:
+            new_data.append({"ip": region["servers"][protocol_type][0]["ip"], "cn": region["servers"][protocol_type][0]["cn"], "id": region["id"],
+                             "name": region["name"], "country": region["country"], "dns": region["dns"], "port_forward": str(region["port_forward"]), "protocol": protocol_type})
+
+
+    df = pd.DataFrame(new_data)
+
+    df.to_csv(output_file, index=False)
+
+
+
+def main():
+    logging.basicConfig(
+        stream=sys.stdout,
+        level=logging.DEBUG,
+        format="%(asctime)s:%(levelname)s: %(message)s",
+    )
+    logging.info("Python main function started")
+
+    curr_path = Path.cwd()
+
+    out_path = (
+        curr_path / "master" / "PublicFeeds" / "PIAVPNDaily" / "pia-servers.csv"
+    )
+    try:
+        out_path.parents[0].mkdir(parents=True, exist_ok=False)
+    except FileExistsError:
+        logging.info("Folder is already present")
+    else:
+        logging.info(f"{out_path} Folder was created")
+
+    logging.info(f"Writing csv file to output directory : {out_path}")
+    download_pia_vpn_servers(out_path)
+
+
+if __name__ == "__main__":
+    main()

PublicFeeds/MSFTIPRanges/MSFT_PublicIPs.csv

@@ -8,6 +8,7 @@ Prefix,Type
 4.208.0.0/12, MSFT Public IP Address Block
 4.224.0.0/12, MSFT Public IP Address Block
 4.240.0.0/12, MSFT Public IP Address Block
+<<<<<<< HEAD
 9.141.0.0/16, MSFT Public IP Address Block
 9.145.0.0/16, MSFT Public IP Address Block
 9.146.0.0/16, MSFT Public IP Address Block
@@ -16,6 +17,10 @@ Prefix,Type
 9.169.0.0/16, MSFT Public IP Address Block
 9.205.0.0/16, MSFT Public IP Address Block
 9.223.0.0/16, MSFT Public IP Address Block
+=======
+9.145.0.0/16, MSFT Public IP Address Block
+9.146.0.0/16, MSFT Public IP Address Block
+>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
 9.234.0.0/15, MSFT Public IP Address Block
 13.64.0.0/11, MSFT Public IP Address Block
 13.96.0.0/13, MSFT Public IP Address Block
@@ -65,12 +70,15 @@ Prefix,Type
 48.208.0.0/13, MSFT Public IP Address Block
 48.216.0.0/14, MSFT Public IP Address Block
 48.220.0.0/15, MSFT Public IP Address Block
+<<<<<<< HEAD
 48.222.0.0/15, MSFT Public IP Address Block
 50.20.64.0/19, MSFT Public IP Address Block
 50.20.128.0/18, MSFT Public IP Address Block
 50.21.32.0/19, MSFT Public IP Address Block
 50.85.0.0/16, MSFT Public IP Address Block
 51.1.0.0/16, MSFT Public IP Address Block
+=======
+>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
 51.4.0.0/15, MSFT Public IP Address Block
 51.8.0.0/16, MSFT Public IP Address Block
 51.10.0.0/15, MSFT Public IP Address Block
@@ -133,7 +141,10 @@ Prefix,Type
 72.54.0.0/16, MSFT Public IP Address Block
 72.144.0.0/14, MSFT Public IP Address Block
 72.152.0.0/14, MSFT Public IP Address Block
+<<<<<<< HEAD
 74.7.0.0/16, MSFT Public IP Address Block
+=======
+>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
 74.144.0.0/12, MSFT Public IP Address Block
 74.160.0.0/14, MSFT Public IP Address Block
 74.176.0.0/14, MSFT Public IP Address Block
@@ -142,13 +153,19 @@ Prefix,Type
 74.240.0.0/14, MSFT Public IP Address Block
 74.248.0.0/15, MSFT Public IP Address Block
 82.87.0.0/16, MSFT Public IP Address Block
+<<<<<<< HEAD
 82.171.0.0/16, MSFT Public IP Address Block
 84.81.0.0/16, MSFT Public IP Address Block
+=======
+>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
 84.222.0.0/16, MSFT Public IP Address Block
 84.223.0.0/16, MSFT Public IP Address Block
 85.210.0.0/15, MSFT Public IP Address Block
 85.212.0.0/16, MSFT Public IP Address Block
+<<<<<<< HEAD
 86.91.0.0/16, MSFT Public IP Address Block
+=======
+>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
 91.190.216.0/21, MSFT Public IP Address Block
 94.245.64.0/18, MSFT Public IP Address Block
 98.64.0.0/14, MSFT Public IP Address Block
@@ -177,8 +194,11 @@ Prefix,Type
 128.251.0.0/16, MSFT Public IP Address Block
 129.75.0.0/16, MSFT Public IP Address Block
 129.135.0.0/16, MSFT Public IP Address Block
+<<<<<<< HEAD
 130.33.0.0/16, MSFT Public IP Address Block
 130.107.0.0/16, MSFT Public IP Address Block
+=======
+>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
 130.131.0.0/16, MSFT Public IP Address Block
 130.213.0.0/16, MSFT Public IP Address Block
 131.107.0.0/16, MSFT Public IP Address Block
@@ -217,6 +237,7 @@ Prefix,Type
 135.13.0.0/16, MSFT Public IP Address Block
 135.18.0.0/16, MSFT Public IP Address Block
 135.85.0.0/16, MSFT Public IP Address Block
+<<<<<<< HEAD
 135.86.0.0/16, MSFT Public IP Address Block
 135.88.0.0/16, MSFT Public IP Address Block
 135.93.0.0/16, MSFT Public IP Address Block
@@ -224,11 +245,18 @@ Prefix,Type
 135.114.0.0/16, MSFT Public IP Address Block
 135.116.0.0/16, MSFT Public IP Address Block
 135.117.0.0/16, MSFT Public IP Address Block
+=======
+135.88.0.0/16, MSFT Public IP Address Block
+135.93.0.0/16, MSFT Public IP Address Block
+135.114.0.0/16, MSFT Public IP Address Block
+135.116.0.0/16, MSFT Public IP Address Block
+>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
 135.118.0.0/16, MSFT Public IP Address Block
 135.119.0.0/16, MSFT Public IP Address Block
 135.130.0.0/16, MSFT Public IP Address Block
 135.149.0.0/16, MSFT Public IP Address Block
 135.171.0.0/16, MSFT Public IP Address Block
+<<<<<<< HEAD
 135.183.0.0/16, MSFT Public IP Address Block
 135.185.0.0/16, MSFT Public IP Address Block
 135.220.0.0/16, MSFT Public IP Address Block
@@ -237,6 +265,13 @@ Prefix,Type
 135.224.0.0/15, MSFT Public IP Address Block
 135.226.0.0/16, MSFT Public IP Address Block
 135.228.0.0/16, MSFT Public IP Address Block
+=======
+135.185.0.0/16, MSFT Public IP Address Block
+135.220.0.0/16, MSFT Public IP Address Block
+135.222.0.0/16, MSFT Public IP Address Block
+135.224.0.0/15, MSFT Public IP Address Block
+135.226.0.0/16, MSFT Public IP Address Block
+>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
 135.229.0.0/16, MSFT Public IP Address Block
 135.230.0.0/16, MSFT Public IP Address Block
 135.232.0.0/14, MSFT Public IP Address Block
@@ -244,9 +279,12 @@ Prefix,Type
 135.240.0.0/16, MSFT Public IP Address Block
 135.241.0.0/16, MSFT Public IP Address Block
 135.244.0.0/16, MSFT Public IP Address Block
+<<<<<<< HEAD
 135.246.0.0/16, MSFT Public IP Address Block
 135.247.0.0/16, MSFT Public IP Address Block
 135.254.0.0/16, MSFT Public IP Address Block
+=======
+>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
 135.255.0.0/16, MSFT Public IP Address Block
 137.116.0.0/15, MSFT Public IP Address Block
 137.135.0.0/16, MSFT Public IP Address Block
@@ -264,10 +302,13 @@ Prefix,Type
 143.209.0.0/16, MSFT Public IP Address Block
 143.226.0.0/16, MSFT Public IP Address Block
 143.241.0.0/16, MSFT Public IP Address Block
+<<<<<<< HEAD
 145.129.0.0/16, MSFT Public IP Address Block
 145.130.0.0/16, MSFT Public IP Address Block
 145.132.0.0/15, MSFT Public IP Address Block
 145.190.0.0/15, MSFT Public IP Address Block
+=======
+>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
 146.147.0.0/16, MSFT Public IP Address Block
 147.145.0.0/16, MSFT Public IP Address Block
 147.214.0.0/16, MSFT Public IP Address Block