зеркало из https://github.com/microsoft/mstic.git
Merge branch 'master' of https://github.com/HarelKatz/mstic into HarelKatz-master
This commit is contained in:
Ashwin Patil
Коммит
8c951a7f9a
|
|
@ -0,0 +1,44 @@
|
|||
name: Daily PIA VPN Servers Feed
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 0 * * *" # Runs at 00:00 AM (UTC) everyday (Check https://crontab.guru/#0_0_*_*_*)
|
||||
|
||||
jobs:
|
||||
download-piavpnservers:
|
||||
name: Download PIA VPN Servers via API
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 🍽️ Checkout the repo
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
path: master
|
||||
fetch-depth: 0
|
||||
|
||||
- name: 🐍 Set up Python 3.8
|
||||
uses: actions/setup-python@v3
|
||||
with:
|
||||
python-version: "3.8"
|
||||
architecture: "x64"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install requests pandas
|
||||
|
||||
- name: 🚀 Run automation script
|
||||
run: python master/.script/get-piavpnservers.py
|
||||
|
||||
- name: Commit files
|
||||
run: |
|
||||
cd master
|
||||
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --local user.name "github-actions[bot]"
|
||||
git add *
|
||||
git commit -m "Adding new PIA vpn servers daily feed" -a
|
||||
|
||||
- name: Push changes
|
||||
uses: ad-m/github-push-action@master
|
||||
with:
|
||||
directory: "master"
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
import requests
|
||||
import json
|
||||
import sys
|
||||
import logging
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def download_pia_vpn_servers(output_file):
|
||||
|
||||
new_data = []
|
||||
|
||||
|
||||
# V4
|
||||
logging.info(f"Downloading PIA VPN server list using API from https://serverlist.piaservers.net/vpninfo/servers/v4")
|
||||
|
||||
|
||||
r_v4 = requests.get("https://serverlist.piaservers.net/vpninfo/servers/v4")
|
||||
|
||||
|
||||
my_json_v4 = r_v4.content.decode("utf8")
|
||||
|
||||
# Remove everything after the last brace
|
||||
my_json_v4 = my_json_v4[:my_json_v4.rfind('}')+1]
|
||||
|
||||
|
||||
data_v4 = json.loads(my_json_v4)["regions"]
|
||||
|
||||
for region in data_v4:
|
||||
for protocol_type in region["servers"]:
|
||||
new_data.append({"ip": region["servers"][protocol_type][0]["ip"], "cn": region["servers"][protocol_type][0]["cn"], "id": region["id"],
|
||||
"name": region["name"], "country": region["country"], "dns": region["dns"], "port_forward": str(region["port_forward"]), "protocol": protocol_type, "offline": ""})
|
||||
|
||||
|
||||
"----------------------------------------------------------------------------"
|
||||
|
||||
|
||||
# V5
|
||||
logging.info(f"Downloading PIA VPN server list using API from https://serverlist.piaservers.net/vpninfo/servers/v5")
|
||||
|
||||
|
||||
r_v5 = requests.get("https://serverlist.piaservers.net/vpninfo/servers/v5")
|
||||
|
||||
|
||||
my_json_v5 = r_v5.content.decode("utf8")
|
||||
|
||||
# Remove everything after the last brace
|
||||
my_json_v5 = my_json_v5[:my_json_v5.rfind('}')+1]
|
||||
|
||||
|
||||
data_v5 = json.loads(my_json_v5)["regions"]
|
||||
|
||||
for region in data_v5:
|
||||
for protocol_type in region["servers"]:
|
||||
new_data.append({"ip": region["servers"][protocol_type][0]["ip"], "cn": region["servers"][protocol_type][0]["cn"], "id": region["id"],
|
||||
"name": region["name"], "country": region["country"], "dns": region["dns"], "port_forward": str(region["port_forward"]), "protocol": protocol_type, "offline": region["offline"] })
|
||||
|
||||
|
||||
"----------------------------------------------------------------------------"
|
||||
|
||||
|
||||
# V6
|
||||
logging.info(f"Downloading PIA VPN server list using API from https://serverlist.piaservers.net/vpninfo/servers/v6")
|
||||
|
||||
|
||||
r_v6 = requests.get("https://serverlist.piaservers.net/vpninfo/servers/v6")
|
||||
|
||||
|
||||
my_json_v6 = r_v6.content.decode("utf8")
|
||||
|
||||
# Remove everything after the last brace
|
||||
my_json_v6 = my_json_v6[:my_json_v6.rfind('}')+1]
|
||||
|
||||
data_v6 = json.loads(my_json_v6)["regions"]
|
||||
|
||||
for region in data_v6:
|
||||
for protocol_type in region["servers"]:
|
||||
new_data.append({"ip": region["servers"][protocol_type][0]["ip"], "cn": region["servers"][protocol_type][0]["cn"], "id": region["id"],
|
||||
"name": region["name"], "country": region["country"], "dns": region["dns"], "port_forward": str(region["port_forward"]), "protocol": protocol_type})
|
||||
|
||||
|
||||
df = pd.DataFrame(new_data)
|
||||
|
||||
df.to_csv(output_file, index=False)
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(
|
||||
stream=sys.stdout,
|
||||
level=logging.DEBUG,
|
||||
format="%(asctime)s:%(levelname)s: %(message)s",
|
||||
)
|
||||
logging.info("Python main function started")
|
||||
|
||||
curr_path = Path.cwd()
|
||||
|
||||
out_path = (
|
||||
curr_path / "master" / "PublicFeeds" / "PIAVPNDaily" / "pia-servers.csv"
|
||||
)
|
||||
try:
|
||||
out_path.parents[0].mkdir(parents=True, exist_ok=False)
|
||||
except FileExistsError:
|
||||
logging.info("Folder is already present")
|
||||
else:
|
||||
logging.info(f"{out_path} Folder was created")
|
||||
|
||||
logging.info(f"Writing csv file to output directory : {out_path}")
|
||||
download_pia_vpn_servers(out_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -8,6 +8,7 @@ Prefix,Type
|
|||
4.208.0.0/12, MSFT Public IP Address Block
|
||||
4.224.0.0/12, MSFT Public IP Address Block
|
||||
4.240.0.0/12, MSFT Public IP Address Block
|
||||
<<<<<<< HEAD
|
||||
9.141.0.0/16, MSFT Public IP Address Block
|
||||
9.145.0.0/16, MSFT Public IP Address Block
|
||||
9.146.0.0/16, MSFT Public IP Address Block
|
||||
|
|
@ -16,6 +17,10 @@ Prefix,Type
|
|||
9.169.0.0/16, MSFT Public IP Address Block
|
||||
9.205.0.0/16, MSFT Public IP Address Block
|
||||
9.223.0.0/16, MSFT Public IP Address Block
|
||||
=======
|
||||
9.145.0.0/16, MSFT Public IP Address Block
|
||||
9.146.0.0/16, MSFT Public IP Address Block
|
||||
>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
|
||||
9.234.0.0/15, MSFT Public IP Address Block
|
||||
13.64.0.0/11, MSFT Public IP Address Block
|
||||
13.96.0.0/13, MSFT Public IP Address Block
|
||||
|
|
@ -65,12 +70,15 @@ Prefix,Type
|
|||
48.208.0.0/13, MSFT Public IP Address Block
|
||||
48.216.0.0/14, MSFT Public IP Address Block
|
||||
48.220.0.0/15, MSFT Public IP Address Block
|
||||
<<<<<<< HEAD
|
||||
48.222.0.0/15, MSFT Public IP Address Block
|
||||
50.20.64.0/19, MSFT Public IP Address Block
|
||||
50.20.128.0/18, MSFT Public IP Address Block
|
||||
50.21.32.0/19, MSFT Public IP Address Block
|
||||
50.85.0.0/16, MSFT Public IP Address Block
|
||||
51.1.0.0/16, MSFT Public IP Address Block
|
||||
=======
|
||||
>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
|
||||
51.4.0.0/15, MSFT Public IP Address Block
|
||||
51.8.0.0/16, MSFT Public IP Address Block
|
||||
51.10.0.0/15, MSFT Public IP Address Block
|
||||
|
|
@ -133,7 +141,10 @@ Prefix,Type
|
|||
72.54.0.0/16, MSFT Public IP Address Block
|
||||
72.144.0.0/14, MSFT Public IP Address Block
|
||||
72.152.0.0/14, MSFT Public IP Address Block
|
||||
<<<<<<< HEAD
|
||||
74.7.0.0/16, MSFT Public IP Address Block
|
||||
=======
|
||||
>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
|
||||
74.144.0.0/12, MSFT Public IP Address Block
|
||||
74.160.0.0/14, MSFT Public IP Address Block
|
||||
74.176.0.0/14, MSFT Public IP Address Block
|
||||
|
|
@ -142,13 +153,19 @@ Prefix,Type
|
|||
74.240.0.0/14, MSFT Public IP Address Block
|
||||
74.248.0.0/15, MSFT Public IP Address Block
|
||||
82.87.0.0/16, MSFT Public IP Address Block
|
||||
<<<<<<< HEAD
|
||||
82.171.0.0/16, MSFT Public IP Address Block
|
||||
84.81.0.0/16, MSFT Public IP Address Block
|
||||
=======
|
||||
>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
|
||||
84.222.0.0/16, MSFT Public IP Address Block
|
||||
84.223.0.0/16, MSFT Public IP Address Block
|
||||
85.210.0.0/15, MSFT Public IP Address Block
|
||||
85.212.0.0/16, MSFT Public IP Address Block
|
||||
<<<<<<< HEAD
|
||||
86.91.0.0/16, MSFT Public IP Address Block
|
||||
=======
|
||||
>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
|
||||
91.190.216.0/21, MSFT Public IP Address Block
|
||||
94.245.64.0/18, MSFT Public IP Address Block
|
||||
98.64.0.0/14, MSFT Public IP Address Block
|
||||
|
|
@ -177,8 +194,11 @@ Prefix,Type
|
|||
128.251.0.0/16, MSFT Public IP Address Block
|
||||
129.75.0.0/16, MSFT Public IP Address Block
|
||||
129.135.0.0/16, MSFT Public IP Address Block
|
||||
<<<<<<< HEAD
|
||||
130.33.0.0/16, MSFT Public IP Address Block
|
||||
130.107.0.0/16, MSFT Public IP Address Block
|
||||
=======
|
||||
>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
|
||||
130.131.0.0/16, MSFT Public IP Address Block
|
||||
130.213.0.0/16, MSFT Public IP Address Block
|
||||
131.107.0.0/16, MSFT Public IP Address Block
|
||||
|
|
@ -217,6 +237,7 @@ Prefix,Type
|
|||
135.13.0.0/16, MSFT Public IP Address Block
|
||||
135.18.0.0/16, MSFT Public IP Address Block
|
||||
135.85.0.0/16, MSFT Public IP Address Block
|
||||
<<<<<<< HEAD
|
||||
135.86.0.0/16, MSFT Public IP Address Block
|
||||
135.88.0.0/16, MSFT Public IP Address Block
|
||||
135.93.0.0/16, MSFT Public IP Address Block
|
||||
|
|
@ -224,11 +245,18 @@ Prefix,Type
|
|||
135.114.0.0/16, MSFT Public IP Address Block
|
||||
135.116.0.0/16, MSFT Public IP Address Block
|
||||
135.117.0.0/16, MSFT Public IP Address Block
|
||||
=======
|
||||
135.88.0.0/16, MSFT Public IP Address Block
|
||||
135.93.0.0/16, MSFT Public IP Address Block
|
||||
135.114.0.0/16, MSFT Public IP Address Block
|
||||
135.116.0.0/16, MSFT Public IP Address Block
|
||||
>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
|
||||
135.118.0.0/16, MSFT Public IP Address Block
|
||||
135.119.0.0/16, MSFT Public IP Address Block
|
||||
135.130.0.0/16, MSFT Public IP Address Block
|
||||
135.149.0.0/16, MSFT Public IP Address Block
|
||||
135.171.0.0/16, MSFT Public IP Address Block
|
||||
<<<<<<< HEAD
|
||||
135.183.0.0/16, MSFT Public IP Address Block
|
||||
135.185.0.0/16, MSFT Public IP Address Block
|
||||
135.220.0.0/16, MSFT Public IP Address Block
|
||||
|
|
@ -237,6 +265,13 @@ Prefix,Type
|
|||
135.224.0.0/15, MSFT Public IP Address Block
|
||||
135.226.0.0/16, MSFT Public IP Address Block
|
||||
135.228.0.0/16, MSFT Public IP Address Block
|
||||
=======
|
||||
135.185.0.0/16, MSFT Public IP Address Block
|
||||
135.220.0.0/16, MSFT Public IP Address Block
|
||||
135.222.0.0/16, MSFT Public IP Address Block
|
||||
135.224.0.0/15, MSFT Public IP Address Block
|
||||
135.226.0.0/16, MSFT Public IP Address Block
|
||||
>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
|
||||
135.229.0.0/16, MSFT Public IP Address Block
|
||||
135.230.0.0/16, MSFT Public IP Address Block
|
||||
135.232.0.0/14, MSFT Public IP Address Block
|
||||
|
|
@ -244,9 +279,12 @@ Prefix,Type
|
|||
135.240.0.0/16, MSFT Public IP Address Block
|
||||
135.241.0.0/16, MSFT Public IP Address Block
|
||||
135.244.0.0/16, MSFT Public IP Address Block
|
||||
<<<<<<< HEAD
|
||||
135.246.0.0/16, MSFT Public IP Address Block
|
||||
135.247.0.0/16, MSFT Public IP Address Block
|
||||
135.254.0.0/16, MSFT Public IP Address Block
|
||||
=======
|
||||
>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
|
||||
135.255.0.0/16, MSFT Public IP Address Block
|
||||
137.116.0.0/15, MSFT Public IP Address Block
|
||||
137.135.0.0/16, MSFT Public IP Address Block
|
||||
|
|
@ -264,10 +302,13 @@ Prefix,Type
|
|||
143.209.0.0/16, MSFT Public IP Address Block
|
||||
143.226.0.0/16, MSFT Public IP Address Block
|
||||
143.241.0.0/16, MSFT Public IP Address Block
|
||||
<<<<<<< HEAD
|
||||
145.129.0.0/16, MSFT Public IP Address Block
|
||||
145.130.0.0/16, MSFT Public IP Address Block
|
||||
145.132.0.0/15, MSFT Public IP Address Block
|
||||
145.190.0.0/15, MSFT Public IP Address Block
|
||||
=======
|
||||
>>>>>>> 2ef619dc293cbe92c940d6409916e0085e68d3bc
|
||||
146.147.0.0/16, MSFT Public IP Address Block
|
||||
147.145.0.0/16, MSFT Public IP Address Block
|
||||
147.214.0.0/16, MSFT Public IP Address Block
|
||||
|
|
|
|||
|
Не удается отобразить этот файл, потому что он имеет неправильное количество полей в строке 11.
|
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
Загрузка…
Ссылка в новой задаче