User/vchintalapat/monitor versions (#2499)

* added scripts to webscrape versions

* added scripts to webscrape versions

* updated update_constants.sh file

* updated update_constants.sh file

* updated monitor_variables.sh file

* updated monitor_variables.sh file

* updated monitor_variables.sh file

* reading from variable group and updating the constants.yml

* updated update_constants.sh

* updated update_constants.sh

* updated update_constants.sh

* modified update_constants.sh

* created generated_files folder

* sorted the keys of latest_stack_versions.yml

* removed generated files folder and temo constants file

* created description file for pr

* added debian flavors to constants.yaml

* modified  description for PR

* modified  description for PR

* modified  description for PR

* added createPR workflow

* added createPR workflow

* Create PRcreation.yml

* Create manual.yml

* Removed createPR workflow

* Removed unnecessary workflows

* added versions to build for node

* added versions to build code for node

* added gpg keys to override_constants.yaml

* organised the code

* organised the code

* removed extra spaces from versionsToBuild files

* added debug statements

* updated versions to build files

* scraped dotnet sdk's

* updated dotnet sdk's in versionsToBuild

* updated dotnet sdk's in versionsToBuild

* removing extra lines in versions to build

* sorting the versions to build file

* removed unnecessary comments and corrected edge cases

* removed temporary constants

* removed temporary constants

* removed latest_versions yaml file

* removed sample files

* passed PIP version

* modified latest dotnetcoresdk version for tests

* hard coded dotnet latest versions

* hard coded dotnet latest versions

* modified tests

* modified sdk versions

* used eol api to get versions

* modified sdk format

* removed apt-get commands

* used relative paths in update_constants.sh
This commit is contained in:
Saipriya-1144 2024-09-10 15:55:28 +05:30 коммит произвёл GitHub
Родитель 931007fa4c
Коммит af6c2414ff
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
17 изменённых файлов: 613 добавлений и 51 удалений

3
.gitignore поставляемый
Просмотреть файл

@ -269,3 +269,6 @@ src/startupscriptgenerator/src/**/__debug_bin
src/startupscriptgenerator/src/**/run.sh
!**/dynamic/Dockerfile
!**/dynamic/buster.Dockerfile
# needed just for pr description
Updated_Values.txt

Просмотреть файл

@ -1,4 +1,36 @@
variables:
NODE_APP_INSIGHTS_SDK_VERSION: 1.8.7
NPM_VERSION: 10.7.0
PM2_VERSION: 5.3.1
YARN_MAJOR_VERSION: "1"
YARN_MINOR_VERSION: "1.22"
YARN_VERSION: 1.22.15
USER_DOTNET_AI_VERSION: 2.8.42
AI_CONNECTION_STRING: 1234(RANDOMVALUE)
SDK_STORAGE_BASE_URL_VALUE: https://oryxsdksdev.blob.core.windows.net
node18DebianFlavors: bullseye
node20DebianFlavors: bullseye,bookworm
python38DebianFlavors: buster,bullseye
python39DebianFlavors: buster,bullseye
python310DebianFlavors: buster,bullseye
python311DebianFlavors: bullseye,bookworm
python312DebianFlavors: bullseye,bookworm
dotnet60DebianFlavors: buster,bullseye
dotnet70DebianFlavors: buster,bullseye
dotnet80DebianFlavors: bullseye,bookworm
dotnet90DebianFlavors: bookworm
php81DebianFlavors: bullseye,buster
php82DebianFlavors: bullseye,buster
php83DebianFlavors: bullseye,buster,bookworm
debianFlavors: 'buster,bullseye,bookworm'
dotnetDebianFlavors: 'bookworm,bullseye,buster'
nodejsDebianFlavors: 'bookworm,bullseye'
phpDebianFlavors: 'bookworm,bullseye,buster'
pythonDebianFlavors: 'bookworm,bullseye,buster'
DotNetCoreAgent_version: 2.8.42
GitHubActions_BaseImage_Buster: mcr.microsoft.com/mirror/docker/library/buildpack-deps@sha256:be7137ba47bdf421d3950abe1200322631e8db11b468cd6f0ebf3d687e556c90
GitHubActions_BaseImage_Bullseye: mcr.microsoft.com/mirror/docker/library/buildpack-deps@sha256:25c86292946cce832340fe808ac47e5a845b68c488f177a47ff44fd59040397d
GitHubActions_BaseImage_Bookworm: mcr.microsoft.com/mirror/docker/library/buildpack-deps@sha256:ff838f535d6e3bace9ab7eefa360d648ba529f9aa57fdd709335f8ef0516cdde
ASPNET_CORE_APP_60: 6.0.32
ASPNET_CORE_APP_60_SHA: 1849c0073f12477b94357a1afb1cbd4ad67764263528b66037c19d554df41e681e4b41c0804b106319fe661d0bc3bae9e29e4913c0d0df33861cf6f32ebaac96
ASPNET_CORE_APP_70: 7.0.20
@ -15,44 +47,16 @@ variables:
NET_CORE_APP_80_SHA: 88e9ac34ad5ac76eec5499f2eb8d1aa35076518c842854ec1053953d34969c7bf1c5b2dbce245dbace3a18c3b8a4c79d2ef2d2ff105ce9d17cbbdbe813d8b16f
NET_CORE_APP_90: 9.0.0-preview.6.24327.7
NET_CORE_APP_90_SHA: 09aa8c4e6ae3ada1a265a5cd2b46779a763163e4dd9a1892b44606b89cf147339e10b7c584dbcaf5404af0553f0ef6c5801436c217f4fe1a5d3bdb6d74aef1d1
NODE_APP_INSIGHTS_SDK_VERSION: 1.8.7
node18Version: 18.20.4
node20Version: 20.15.1
NPM_VERSION: 10.7.0
php81Version: 8.1.29
php81Version_SHA: 288884af60581d4284baba2ace9ca6d646f72facbd3e3c2dd2acc7fe6f903536
php82Version: 8.2.21
php82Version_SHA: 8cc44d51bb2506399ec176f70fe110f0c9e1f7d852a5303a2cd1403402199707
php83Version: 8.3.9
php83Version_SHA: bf4d7b8ea60a356064f88485278bd6f941a230ec16f0fc401574ce1445ad6c77
PM2_VERSION: 5.3.1
python310Version: 3.10.14
python311Version: 3.11.8
python312Version: 3.12.2
python38Version: 3.8.19
python39Version: 3.9.19
YARN_MAJOR_VERSION: 1
YARN_MINOR_VERSION: 1.22
YARN_VERSION: 1.22.15
USER_DOTNET_AI_VERSION: 2.8.42
AI_CONNECTION_STRING: "1234(RANDOMVALUE)"
SDK_STORAGE_BASE_URL_VALUE: 'https://oryxsdksdev.blob.core.windows.net'
node18DebianFlavors: 'bullseye'
node20DebianFlavors: 'bullseye,bookworm'
python38DebianFlavors: 'buster,bullseye'
python39DebianFlavors: 'buster,bullseye'
python310DebianFlavors: 'buster,bullseye'
python311DebianFlavors: 'bullseye,bookworm'
python312DebianFlavors: 'bullseye,bookworm'
dotnet60DebianFlavors: 'buster,bullseye'
dotnet70DebianFlavors: 'buster,bullseye'
dotnet80DebianFlavors: 'bullseye,bookworm'
dotnet90DebianFlavors: 'bookworm'
php81DebianFlavors: 'bullseye,buster'
php82DebianFlavors: 'bullseye,buster'
php83DebianFlavors: 'bullseye,buster,bookworm'
DotNetCoreAgent_version: 2.8.42
GitHubActions_BaseImage_Buster: 'mcr.microsoft.com/mirror/docker/library/buildpack-deps@sha256:be7137ba47bdf421d3950abe1200322631e8db11b468cd6f0ebf3d687e556c90'
GitHubActions_BaseImage_Bullseye: 'mcr.microsoft.com/mirror/docker/library/buildpack-deps@sha256:25c86292946cce832340fe808ac47e5a845b68c488f177a47ff44fd59040397d'
GitHubActions_BaseImage_Bookworm: 'mcr.microsoft.com/mirror/docker/library/buildpack-deps@sha256:ff838f535d6e3bace9ab7eefa360d648ba529f9aa57fdd709335f8ef0516cdde'
python39Version: 3.9.19

Просмотреть файл

@ -0,0 +1,58 @@
# #!/bin/bash
pip install bs4 lxml requests
set -e
error_handler() {
echo "Error occurred in script at line: $1"
exit 1
}
trap 'error_handler $LINENO' ERR
mkdir -p generated_files
curl -o generated_files/php_version.xml "https://www.php.net/downloads.php"
curl -o generated_files/node_version.xml "https://nodejs.org/en/about/previous-releases"
curl -o generated_files/dotnet_version.xml "https://dotnet.microsoft.com/en-us/download/dotnet"
create_versionfile() {
FILE=$1
if [ ! -e "$FILE" ]; then
# Create the file
echo -n "" > $FILE
echo "File '$FILE' created."
else
echo "File '$FILE' already exists."
fi
}
create_versionfile generated_files/node_latest_versions.txt
create_versionfile generated_files/php_latest_versions.txt
create_versionfile generated_files/python_latest_versions.txt
create_versionfile generated_files/dotnet_latest_versions.txt
create_versionfile generated_files/dotnet_sdk_latest_versions.txt
latest_stack_versions_FILE=latest_stack_versions.yaml
cat <<EOL > $latest_stack_versions_FILE
variables:
EOL
python3 web_scrap_files/php_versions.py
python3 web_scrap_files/node_versions.py
python3 web_scrap_files/python_versions.py
python3 web_scrap_files/dotnet_versions.py
chmod +x update_latest_stack_versions.sh
./update_latest_stack_versions.sh
chmod +x update_constants.sh
./update_constants.sh
chmod +x update_versions_to_build.sh
./update_versions_to_build.sh
rm -rf "generated_files"
rm "Stack_Updated_values.txt"
rm "latest_stack_versions.yaml"

Просмотреть файл

@ -0,0 +1,46 @@
variables:
NODE_APP_INSIGHTS_SDK_VERSION: 1.8.7
NPM_VERSION: 10.7.0
PM2_VERSION: 5.3.1
YARN_MAJOR_VERSION: "1"
YARN_MINOR_VERSION: "1.22"
YARN_VERSION: 1.22.15
USER_DOTNET_AI_VERSION: 2.8.42
AI_CONNECTION_STRING: 1234(RANDOMVALUE)
SDK_STORAGE_BASE_URL_VALUE: https://oryxsdksdev.blob.core.windows.net
node18DebianFlavors: bullseye
node20DebianFlavors: bullseye,bookworm
python38DebianFlavors: buster,bullseye
python39DebianFlavors: buster,bullseye
python310DebianFlavors: buster,bullseye
python311DebianFlavors: bullseye,bookworm
python312DebianFlavors: bullseye,bookworm
dotnet60DebianFlavors: buster,bullseye
dotnet70DebianFlavors: buster,bullseye
dotnet80DebianFlavors: bullseye,bookworm
dotnet90DebianFlavors: bookworm
php81DebianFlavors: bullseye,buster
php82DebianFlavors: bullseye,buster
php83DebianFlavors: bullseye,buster,bookworm
debianFlavors: 'buster,bullseye,bookworm'
dotnetDebianFlavors: 'bookworm,bullseye,buster'
nodejsDebianFlavors: 'bookworm,bullseye'
phpDebianFlavors: 'bookworm,bullseye,buster'
pythonDebianFlavors: 'bookworm,bullseye,buster'
DotNetCoreAgent_version: 2.8.42
GitHubActions_BaseImage_Buster: mcr.microsoft.com/mirror/docker/library/buildpack-deps@sha256:be7137ba47bdf421d3950abe1200322631e8db11b468cd6f0ebf3d687e556c90
GitHubActions_BaseImage_Bullseye: mcr.microsoft.com/mirror/docker/library/buildpack-deps@sha256:25c86292946cce832340fe808ac47e5a845b68c488f177a47ff44fd59040397d
GitHubActions_BaseImage_Bookworm: mcr.microsoft.com/mirror/docker/library/buildpack-deps@sha256:ff838f535d6e3bace9ab7eefa360d648ba529f9aa57fdd709335f8ef0516cdde
debianFlavors: 'buster,bullseye,bookworm'
dotnetDebianFlavors: 'bookworm,bullseye,buster'
nodejsDebianFlavors: 'bookworm,bullseye'
phpDebianFlavors: 'bookworm,bullseye,buster'
pythonDebianFlavors: 'bookworm,bullseye,buster'
php81_GPG_keys: '528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544'
php82_GPG_keys: '1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544'
php83_GPG_keys: '1198C0117593497A5EC5C199286AF1F9897469DC AFD8691FDAEDF03BDF6E460563F15A9B715376CA'
python310_GPG_keys: 'A035C8C19219BA821ECEA86B64E628F8D684696D'
python311_GPG_keys: 'A035C8C19219BA821ECEA86B64E628F8D684696D'
python312_GPG_keys: '7169605F62C751356D054A26A821E680E5FA6305'
python38_GPG_keys: 'E3FF2839C048B25C084DEBE9B26995E310250568'
python39_GPG_keys: 'E3FF2839C048B25C084DEBE9B26995E310250568'

Просмотреть файл

@ -0,0 +1,113 @@
#!/bin/bash
Old_constants_FILE="../images/constants.yml"
Temp_constants_FILE="../images/temp_constants.yml"
mv $Old_constants_FILE $Temp_constants_FILE
constants_FILE="../images/constants.yml"
cat <<EOL > $constants_FILE
variables:
EOL
Updated_ValuesFILE=Updated_Values.txt
echo -n "" > $Updated_ValuesFILE
Stack_Updated_values=Stack_Updated_values.txt
echo -n "" > $Stack_Updated_values
update_constants_file(){
while IFS= read -r line; do
# Use yq to parse the YAML line and extract the key and value
key=$(echo "$line" | yq e 'keys' - | sed 's/^[[:space:]]*-*//' | sed 's/^[[:space:]]*//')
value=$(echo "$line" | yq e '.[]' -)
echo "Key: $key, Value: $value,"
keyInVariableGroup=$(echo "$key" | tr '[:lower:]' '[:upper:]')
# Check if the key exists in the environment variables
if printenv "$keyInVariableGroup" > /dev/null; then
# If the key exists, get its value
valueInVariableGroup=$(printenv "$keyInVariableGroup")
echo "The value of $key is: $valueInVariableGroup"
if [[ "$valueInVariableGroup" = "latest" ]]; then
old_value=$(yq eval ".variables.$key" $Temp_constants_FILE)
if [ $old_value = $value ]; then
yq eval ".variables.$key = \"$value\"" -i $constants_FILE
echo "$key is already upto date"
else
# during updating php or python versions we need gpg keys as well during sdk building
# so if it is not present in Temp_constants_FILE or in variable group, ask for it
if [[ "$1" = "latest_stack_versions.yaml" && ("$key" = *"python"* || "$key" = *"php"*) && "$key" != *"SHA"* ]]; then
gpgkeys=$(echo "$key" | sed 's/Version.*//')
gpgkeys+="_GPG_keys"
if ! yq eval ".variables | has(\"$gpgkeys\")" $Temp_constants_FILE; then
echo "GPG Keys needed for $key" >&2
exit 1
fi
fi
yq eval ".variables.$key = \"$value\"" -i $constants_FILE
echo "Updated constants.yml with latest value $key=$value"
# This is for all updates (for PR description)
if [[ "$key" != *"SHA"* ]]; then
echo "$key=$value" >> "$Stack_Updated_values"
if [ -n $old_value ]; then
update_line="Updated $key from $old_value to $value"
else
update_line="Added $key to $value"
fi
echo "$update_line" >> "$Updated_ValuesFILE"
fi
fi
elif [[ "$valueInVariableGroup" = "dont_change" ]]; then
old_value=$(yq eval ".variables.$key" $Temp_constants_FILE)
yq eval ".variables.$key = \"$old_value\"" -i $constants_FILE
echo "constants.yml with old value $key=$old_value"
else
old_value=$(yq eval ".variables.$key" $Temp_constants_FILE)
if [ $old_value = $valueInVariableGroup ]; then
echo "$key has required value in constants.yml"
yq eval ".variables.$key = \"$valueInVariableGroup\"" -i $constants_FILE
else
if [[ "$1" == "override_constants.yaml" ]]; then
yq eval ".variables.$key = \"$valueInVariableGroup\"" -i "$1"
fi
if [[ "$1" == "latest_stack_versions.yaml" && "$key" != *"SHA"* ]]; then
echo "$key=$valueInVariableGroup" >> "$Stack_Updated_values"
fi
yq eval ".variables.$key = \"$valueInVariableGroup\"" -i $constants_FILE
echo "Updated constants.yml with given value $key=$valueInVariableGroup"
# This is for all updates (for PR description)
if [[ "$key" != *"SHA"* ]]; then
if [ -n $old_value ]; then
update_line="Updated $key from $old_value to $value"
else
update_line="Added $key to $value"
fi
echo "$update_line" >> "$Updated_ValuesFILE"
fi
fi
fi
else
yq eval ".variables.$key = \"$value\"" -i $constants_FILE
echo "Added $key = $value in constants.yml"
fi
done < <(yq e '.[]' "$1")
}
update_constants_file "override_constants.yaml"
update_constants_file "latest_stack_versions.yaml"
rm $Temp_constants_FILE

Просмотреть файл

@ -0,0 +1,41 @@
#read the whole text file, if that particular val exists update it in latest_stack_versions.yaml
split_lines() {
while IFS= read -r line; do
if [[ "$line" == *"="* ]]; then
IFS=',' read -ra keyvalue_pairs <<< "$line"
for keyvalue_pair in "${keyvalue_pairs[@]}"; do
key="${keyvalue_pair%%=*}"
value="${keyvalue_pair#*=}"
echo "key: $key, value: $value"
if yq eval ".variables | has(\"$key\")" latest_stack_versions.yaml | grep -q 'true'; then
if [[ "$key" != *"python"* ]]; then
yq eval ".variables.$key = \"$value\"" -i latest_stack_versions.yaml
else
#this is only for python, in https://www.python.org/downloads/ all available minor versions are present of a major version
#so update with latest one
current_value=$(yq eval ".variables.$key" latest_stack_versions.yaml)
# Update the key in latest_stack_versions.yaml
if [[ $(printf '%s\n' "$current_value" "$value" | sort -V | tail -n 1) != "$current_value" ]]; then
yq eval ".variables.$key = \"$value\"" -i latest_stack_versions.yaml
echo "Updated $key in latest_stack_versions.yaml"
fi
fi
else
yq eval ".variables.$key = \"$value\"" -i latest_stack_versions.yaml
echo "Added $key to latest_stack_versions.yaml"
fi
done
fi
done < "$1"
}
split_lines "generated_files/node_latest_versions.txt"
split_lines "generated_files/python_latest_versions.txt"
split_lines "generated_files/php_latest_versions.txt"
split_lines "generated_files/dotnet_latest_versions.txt"
yq eval -i 'sort_keys(..)' "latest_stack_versions.yaml"

Просмотреть файл

@ -0,0 +1,138 @@
sort_versions_to_build_file(){
versionsToBuild_File="$1"
tempfile1=$(mktemp)
tempfile2=$(mktemp)
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ "$line" = *"#"* ]]; then
echo "$line" >> "$tempfile1"
elif [[ "$line" =~ [^[:space:]] ]]; then
echo "$line" >> "$tempfile2"
fi
done < "$versionsToBuild_File"
sort -V $tempfile2 -o $tempfile2
cat $tempfile2 >> $tempfile1
cp $tempfile1 $versionsToBuild_File
}
update_stack_versions_to_build(){
versionsToBuild_FILE="$1"
version="$2"
value="$3"
key="$4"
version_found=false
while IFS= read -r line; do
if [[ "$line" == *"$value"* ]]; then
version_found=true
echo "version already exists in sdks"
break
fi
done < "$versionsToBuild_FILE"
if ! $version_found; then
if [[ "$key" == *"node"* ]]; then
# Check if the last line is empty
if [ -n "$(tail -c 1 "$versionsToBuild_FILE")" ]; then
echo "" >> "$versionsToBuild_FILE"
fi
echo -n "$value" >> "$versionsToBuild_FILE"
updated_files+=("$versionsToBuild_FILE")
# sort_versions_to_build_file "$versionsToBuild_FILE"
elif [[ "$key" == *"python"* ]]; then
gpgkeyname="python${version}_GPG_keys"
gpgkeysvalue=$(yq eval ".variables.$gpgkeyname" override_constants.yaml)
if [ -n "$(tail -c 1 "$versionsToBuild_FILE")" ]; then
echo "" >> "$versionsToBuild_FILE"
fi
echo -n "$value, $gpgkeysvalue," >> "$versionsToBuild_FILE"
updated_files+=("$versionsToBuild_FILE")
# sort_versions_to_build_file "$versionsToBuild_FILE"
elif [[ "$key" == *"php"* ]]; then
gpgkeyname="php${version}_GPG_keys"
gpgkeysvalue=$(yq eval ".variables.$gpgkeyname" override_constants.yaml)
phpSHAName="php${version}Version_SHA"
phpSHAValue=$(yq eval ".variables.$phpSHAName" latest_stack_versions.yaml)
if [ -n "$(tail -c 1 "$versionsToBuild_FILE")" ]; then
echo "" >> "$versionsToBuild_FILE"
fi
echo -n "$value, $phpSHAValue, $gpgkeysvalue," >> "$versionsToBuild_FILE"
updated_files+=("$versionsToBuild_FILE")
# sort_versions_to_build_file "$versionsToBuild_FILE"
fi
fi
}
update_versions_to_build() {
key="$1"
value="$2"
version=${key//[^0-9]/}
if [[ "$key" == *"node"* ]]; then
versionsToBuild_Folder=$(cd .. && pwd)/platforms/nodejs/versions
debianFlavors="node$version"
elif [[ "$key" == *"python"* ]]; then
versionsToBuild_Folder=$(cd .. && pwd)/platforms/python/versions
debianFlavors="python$version"
elif [[ "$key" == *"php"* ]]; then
versionsToBuild_Folder=$(cd .. && pwd)/platforms/php/versions
debianFlavors="php$version"
elif [[ "$key" == *"NET"* ]]; then
versionsToBuild_Folder=$(cd .. && pwd)/platforms/dotnet/versions
debianFlavors="dotnet$version"
fi
debianFlavors+="DebianFlavors"
echo "The one which needs to be searched is $debianFlavors"
alldebianFlavors=$(yq eval ".variables.$debianFlavors" override_constants.yaml)
echo "$alldebianFlavors"
IFS=','
for flavor in $alldebianFlavors; do
echo "$flavor"
versionsToBuild_FILE="$versionsToBuild_Folder/$flavor/versionsToBuild.txt"
if [[ "$key" = *"NET"* ]]; then
while IFS= read -r line; do
echo "Sdk_version line is $line"
if [[ "$line" = *"$value"* ]]; then
sdk_version=$(echo "$line" | sed 's/^[^:]*://')
echo "after processing sdk_version is $sdk_version"
version_found=false
while IFS= read -r line_in_versionsToBuild || [[ -n "$line_in_versionsToBuild" ]]; do
if [[ "$line_in_versionsToBuild" = *"$sdk_version"* ]]; then
version_found=true
fi
done < "$versionsToBuild_FILE"
if ! $version_found; then
if [ -n "$(tail -c 1 "$versionsToBuild_FILE")" ]; then
echo "" >> "$versionsToBuild_FILE"
fi
echo "$sdk_version" >> "$versionsToBuild_FILE"
updated_files+=("$versionsToBuild_FILE")
fi
fi
done < "generated_files/dotnet_sdk_latest_versions.txt"
else
update_stack_versions_to_build $versionsToBuild_FILE $version $value $key
fi
done
}
file="Stack_Updated_values.txt"
# for sorting the updated files later
updated_files=()
# Read the file line by line
while IFS='=' read -r key value; do
# Process each key-value pair
echo "Key: $key, Value: $value"
update_versions_to_build $key $value
done < "$file"
for element in "${updated_files[@]}"; do
sort_versions_to_build_file "$element"
done

Просмотреть файл

@ -0,0 +1,120 @@
from bs4 import BeautifulSoup
import requests
with open('generated_files/dotnet_version.xml','r') as file:
content = file.read()
soup = BeautifulSoup(content, 'lxml-xml')
version_table=soup.select('#supported-versions-table .table tr')
print("Available Dotnet Versions on Web")
def scrape_CheckSum(version,category,type):
runtime_version="".join([category,version])
# type is sdk or runtime
url=f"https://dotnet.microsoft.com/en-us/download/dotnet/thank-you/{type}-{runtime_version}-linux-x64-binaries"
response=requests.get(url)
html_content=response.text
# Parse the HTML content with BeautifulSoup
soup = BeautifulSoup(html_content, 'html.parser')
check_sum=soup.select('#checksum')[0]['value']
return check_sum
def scrap_sdk_versions(HTML_CONTENT,ASPNET_runtime_version,NET_runtime_version):
all_version_tags=HTML_CONTENT.select('h3')
all_description_tags=HTML_CONTENT.select('dl')
for index,version_tag in enumerate(all_version_tags):
if "sdk" in version_tag.get('id'):
sdk_version=(version_tag.text).replace("SDK ","").strip()
check_sum=scrape_CheckSum(sdk_version,"","sdk")
# This is to check for full_version
if(len(all_description_tags)>index):
description_title_tags=all_description_tags[index].select('dt')
for title_tag in description_title_tags:
if "Full version" in title_tag.text:
full_version=title_tag.find_next_sibling().text
sdk_version=full_version
print(f"sdk_version is {sdk_version}, SHA is {check_sum}")
with open('generated_files/dotnet_sdk_latest_versions.txt', 'a') as version_file:
version_file.write(f"{ASPNET_runtime_version}:{sdk_version}, {check_sum},\n")
version_file.write(f"{NET_runtime_version}:{sdk_version}, {check_sum},\n")
return
def scrap_runtime_versions(HTML_CONTENT,major_version):
all_version_tags=HTML_CONTENT.select('h3')
all_description_tags=HTML_CONTENT.select('dl')
x=major_version.replace('.','')
ASPNET_runtime_version=None
NET_runtime_version=None
for index,version_tag in enumerate(all_version_tags):
if ("runtime" in version_tag.get('id')) and not ("desktop" in version_tag.get('id')):
if ("ASP.NET Core Runtime" in version_tag.text):
runtime_version=(version_tag.text).replace("ASP.NET Core Runtime ","").strip()
check_sum=scrape_CheckSum(runtime_version,"aspnetcore-","runtime")
if(len(all_description_tags)>index):
description_title_tags=all_description_tags[index].select('dt')
for title_tag in description_title_tags:
if "Full version" in title_tag.text:
full_version=title_tag.find_next_sibling().text
runtime_version=full_version
ASPNET_runtime_version=runtime_version
print(f"ASP_runtime_version is {runtime_version}, SHA is {check_sum}")
with open('generated_files/dotnet_latest_versions.txt', 'a') as version_file:
version_file.write(f"ASPNET_CORE_APP_{x}={runtime_version},")
version_file.write(f"ASPNET_CORE_APP_{x}_SHA={check_sum}\n")
if (".NET Runtime" in version_tag.text):
runtime_version=(version_tag.text).replace(".NET Runtime ","").strip()
check_sum=scrape_CheckSum(runtime_version,"","runtime")
if(len(all_description_tags)>index):
description_title_tags=all_description_tags[index].select('dt')
for title_tag in description_title_tags:
if "Full version" in title_tag.text:
full_version=title_tag.find_next_sibling().text
runtime_version=full_version
NET_runtime_version=runtime_version
print(f"NET_runtime_version is {runtime_version}, SHA is {check_sum}")
with open('generated_files/dotnet_latest_versions.txt', 'a') as version_file:
version_file.write(f"NET_CORE_APP_{x}={runtime_version},")
version_file.write(f"NET_CORE_APP_{x}_SHA={check_sum}\n")
return [ASPNET_runtime_version,NET_runtime_version]
def scrap_particular_version(major_version,version):
url=f"https://dotnet.microsoft.com/en-us/download/dotnet/{major_version}"
response=requests.get(url)
html_content=response.text
# Parse the HTML content with BeautifulSoup
soup = BeautifulSoup(html_content, 'html.parser')
version_details=soup.select('.download-wrap .row .col-md-6')
runtime_fullversions=scrap_runtime_versions(version_details[1],major_version)
scrap_sdk_versions(version_details[0],runtime_fullversions[0],runtime_fullversions[1])
return
for index,each_version in enumerate(version_table):
if index !=0:
version=each_version.find_all('td')[3].text
split_version=version.split('.')
major_version='.'.join(split_version[:2])
print(f"version is {version}")
scrap_particular_version(major_version,version)

Просмотреть файл

@ -0,0 +1,14 @@
import requests
from datetime import date
response = requests.get('https://endoflife.date/api/nodejs.json')
json_data = response.json()
todays_date = date.today().strftime("%Y-%m-%d")
for element in json_data:
if element["eol"] != True and element["eol"] > todays_date:
version = element["latest"]
major_version = element["cycle"]
with open('generated_files/node_latest_versions.txt', 'a') as version_file:
version_file.write(f"node{major_version}Version={version}\n")

Просмотреть файл

@ -0,0 +1,31 @@
import requests
from datetime import date
from bs4 import BeautifulSoup
with open('generated_files/php_version.xml', 'r') as file:
content = file.read()
soup = BeautifulSoup(content, 'lxml-xml')
def getSHA(php_version):
elements = soup.find("a", href=f"/distributions/php-{php_version}.tar.xz")
if elements:
element_SHA = elements.find_parent().find(class_="sha256").text
return element_SHA
else:
return None
response = requests.get('https://endoflife.date/api/php.json')
json_data = response.json()
todays_date = date.today().strftime("%Y-%m-%d")
for element in json_data:
if element["eol"] != True and element["eol"] > todays_date:
version = element["latest"]
version_SHA = getSHA(element["latest"])
if version_SHA:
x = element["cycle"].replace('.', '')
with open('generated_files/php_latest_versions.txt', 'a') as version_file:
version_file.write(f"php{x}Version={version},")
version_file.write(f"php{x}Version_SHA={version_SHA}\n")

Просмотреть файл

@ -0,0 +1,15 @@
import requests
from datetime import date
response = requests.get('https://endoflife.date/api/python.json')
json_data = response.json()
todays_date = date.today().strftime("%Y-%m-%d")
for element in json_data:
if element["eol"] > todays_date:
version = element["latest"]
major_version = element["cycle"].replace('.', '')
with open('generated_files/python_latest_versions.txt', 'a') as version_file:
version_file.write(f"python{major_version}Version={version}\n")

Просмотреть файл

@ -7,7 +7,6 @@
12.22.9
12.22.11
12.22.12
14.15.0
14.15.1
14.16.0
@ -19,7 +18,6 @@
14.20.1
14.21.2
14.21.3
16.5.0
16.6.1
16.8.0
@ -32,7 +30,6 @@
16.20.0
16.20.1
16.20.2
18.0.0
18.1.0
18.2.0
@ -47,7 +44,6 @@
18.19.1
18.20.3
18.20.4
20.9.0
20.11.0
20.11.1

Просмотреть файл

@ -10,7 +10,6 @@
12.22.9
12.22.11
12.22.12
14.15.0
14.15.1
14.16.0
@ -22,14 +21,12 @@
14.20.1
14.21.2
14.21.3
15.0.0
15.0.1
15.1.0
15.2.0
15.2.1
15.3.0
16.5.0
16.6.1
16.8.0
@ -42,7 +39,6 @@
16.20.0
16.20.1
16.20.2
17.0.1
17.1.0
17.2.0

Просмотреть файл

@ -1,14 +1,8 @@
# version, sha, gpg keys, dockerfile
8.1.22, 9ea4f4cfe775cb5866c057323d6b320f3a6e0adb1be41a068ff7bfec6f83e71d, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.9, 1e6cb77f997613864ab3127fbfc6a8c7fdaa89a95e8ed6167617b913b4de4765, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.16, 28cdc995b7d5421711c7044294885fcde4390c9f67504a994b4cf9bc1b5cc593, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.21, 8cc44d51bb2506399ec176f70fe110f0c9e1f7d852a5303a2cd1403402199707, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.3.4, 39a337036a546e5c28aea76cf424ac172db5156bd8a8fd85252e389409a5ba63, 1198C0117593497A5EC5C199286AF1F9897469DC AFD8691FDAEDF03BDF6E460563F15A9B715376CA,
8.3.8, aea358b56186f943c2bbd350c9005b9359133d47e954cfc561385319ae5bb8d7, 1198C0117593497A5EC5C199286AF1F9897469DC AFD8691FDAEDF03BDF6E460563F15A9B715376CA,
8.3.9, bf4d7b8ea60a356064f88485278bd6f941a230ec16f0fc401574ce1445ad6c77, 1198C0117593497A5EC5C199286AF1F9897469DC AFD8691FDAEDF03BDF6E460563F15A9B715376CA,

Просмотреть файл

@ -9,14 +9,12 @@
7.4.28, 9cc3b6f6217b60582f78566b3814532c4b71d517876c25013ae51811e65d8fce, 42670A7FE4D0441C8E4632349E4FDC074A4EF02D 5A52880781F755608BF815FC910DEB46F53EA312,
7.4.30, ea72a34f32c67e79ac2da7dfe96177f3c451c3eefae5810ba13312ed398ba70d, 42670A7FE4D0441C8E4632349E4FDC074A4EF02D 5A52880781F755608BF815FC910DEB46F53EA312,
7.4.33, 924846abf93bc613815c55dd3f5809377813ac62a9ec4eb3778675b82a27b927, 42670A7FE4D0441C8E4632349E4FDC074A4EF02D 5A52880781F755608BF815FC910DEB46F53EA312,
8.0.22, 130937c0fa3050cd33d6c415402f6ccbf0682ae83eb8d39c91164224ddfe57f1, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.0.25, a291b71d0498707fc5514eb5b9513e88f0f1d4890bcdefd67282ded8a2bfb941, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.0.27, f942cbfe2f7bacbb8039fb79bbec41c76ea779ac5c8157f21e1e0c1b28a5fc3a, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.0.28, 5e07278a1f315a67d36a676c01343ca2d4da5ec5bdb15d018e4248b3012bc0cd, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.0.29, 14db2fbf26c07d0eb2c9fab25dbde7e27726a3e88452cca671f0896bbb683ca9, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.0.30, 216ab305737a5d392107112d618a755dc5df42058226f1670e9db90e77d777d9, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.1.0, a1317eff0723a2b3d3122bbfe107a1158570ea2822dc35a5fb360086db0f6bbc, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.1.4, 05a8c0ac30008154fb38a305560543fc172ba79fb957084a99b8d3b10d5bdb4b, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.1.6, da38d65bb0d5dd56f711cd478204f2b62a74a2c2b0d2d523a78d6eb865b2364c, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
@ -32,7 +30,6 @@
8.1.22, 9ea4f4cfe775cb5866c057323d6b320f3a6e0adb1be41a068ff7bfec6f83e71d, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.1.27, 479e65c3f05714d4aace1370e617d78e49e996ec7a7579a5be47535be61f0658, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.1.29, 288884af60581d4284baba2ace9ca6d646f72facbd3e3c2dd2acc7fe6f903536, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.0, 6ea4c2dfb532950fd712aa2a08c1412a6a81cd1334dd0b0bf88a8e44c2b3a943, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.1, 650d3bd7a056cabf07f6a0f6f1dd8ba45cd369574bbeaa36de7d1ece212c17af, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.2, bdc4aa38e652bac86039601840bae01c0c3653972eaa6f9f93d5f71953a7ee33, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
@ -47,7 +44,6 @@
8.2.17, 1cc4ef733ba58f6557db648012471f1916e5bac316303aa165535bedab08ee35, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.20, 4474cc430febef6de7be958f2c37253e5524d5c5331a7e1765cd2d2234881e50, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.21, 8cc44d51bb2506399ec176f70fe110f0c9e1f7d852a5303a2cd1403402199707, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.3.4, 39a337036a546e5c28aea76cf424ac172db5156bd8a8fd85252e389409a5ba63, 1198C0117593497A5EC5C199286AF1F9897469DC AFD8691FDAEDF03BDF6E460563F15A9B715376CA,
8.3.8, aea358b56186f943c2bbd350c9005b9359133d47e954cfc561385319ae5bb8d7, 1198C0117593497A5EC5C199286AF1F9897469DC AFD8691FDAEDF03BDF6E460563F15A9B715376CA,
8.3.9, bf4d7b8ea60a356064f88485278bd6f941a230ec16f0fc401574ce1445ad6c77, 1198C0117593497A5EC5C199286AF1F9897469DC AFD8691FDAEDF03BDF6E460563F15A9B715376CA,

Просмотреть файл

@ -15,7 +15,6 @@
7.4.28, 9cc3b6f6217b60582f78566b3814532c4b71d517876c25013ae51811e65d8fce, 42670A7FE4D0441C8E4632349E4FDC074A4EF02D 5A52880781F755608BF815FC910DEB46F53EA312,
7.4.30, ea72a34f32c67e79ac2da7dfe96177f3c451c3eefae5810ba13312ed398ba70d, 42670A7FE4D0441C8E4632349E4FDC074A4EF02D 5A52880781F755608BF815FC910DEB46F53EA312,
7.4.33, 924846abf93bc613815c55dd3f5809377813ac62a9ec4eb3778675b82a27b927, 42670A7FE4D0441C8E4632349E4FDC074A4EF02D 5A52880781F755608BF815FC910DEB46F53EA312,
8.0.1, 208b3330af881b44a6a8c6858d569c72db78dab97810332978cc65206b0ec2dc, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.0.2, 84dd6e36f48c3a71ff5dceba375c1f6b34b71d4fa9e06b720780127176468ccc, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.0.3, c9816aa9745a9695672951eaff3a35ca5eddcb9cacf87a4f04b9fb1169010251, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
@ -30,7 +29,6 @@
8.0.28, 5e07278a1f315a67d36a676c01343ca2d4da5ec5bdb15d018e4248b3012bc0cd, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.0.29, 14db2fbf26c07d0eb2c9fab25dbde7e27726a3e88452cca671f0896bbb683ca9, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.0.30, 216ab305737a5d392107112d618a755dc5df42058226f1670e9db90e77d777d9, 1729F83938DA44E27BA0F4D3DBDB397470D12172 BFDDD28642824F8118EF77909B67A5C12229118F,
8.1.0, a1317eff0723a2b3d3122bbfe107a1158570ea2822dc35a5fb360086db0f6bbc, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.1.4, 05a8c0ac30008154fb38a305560543fc172ba79fb957084a99b8d3b10d5bdb4b, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.1.6, da38d65bb0d5dd56f711cd478204f2b62a74a2c2b0d2d523a78d6eb865b2364c, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
@ -46,7 +44,6 @@
8.1.22, 9ea4f4cfe775cb5866c057323d6b320f3a6e0adb1be41a068ff7bfec6f83e71d, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.1.27, 479e65c3f05714d4aace1370e617d78e49e996ec7a7579a5be47535be61f0658, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.1.29, 288884af60581d4284baba2ace9ca6d646f72facbd3e3c2dd2acc7fe6f903536, 528995BFEDFBA7191D46839EF9BA0ADA31CBD89E 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.0, 6ea4c2dfb532950fd712aa2a08c1412a6a81cd1334dd0b0bf88a8e44c2b3a943, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.1, 650d3bd7a056cabf07f6a0f6f1dd8ba45cd369574bbeaa36de7d1ece212c17af, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.2, bdc4aa38e652bac86039601840bae01c0c3653972eaa6f9f93d5f71953a7ee33, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
@ -61,7 +58,6 @@
8.2.17, 1cc4ef733ba58f6557db648012471f1916e5bac316303aa165535bedab08ee35, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.20, 4474cc430febef6de7be958f2c37253e5524d5c5331a7e1765cd2d2234881e50, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.2.21, 8cc44d51bb2506399ec176f70fe110f0c9e1f7d852a5303a2cd1403402199707, 1198C0117593497A5EC5C199286AF1F9897469DC 39B641343D8C104B2B146DC3F9C39DC0B9698544,
8.3.4, 39a337036a546e5c28aea76cf424ac172db5156bd8a8fd85252e389409a5ba63, 1198C0117593497A5EC5C199286AF1F9897469DC AFD8691FDAEDF03BDF6E460563F15A9B715376CA,
8.3.8, aea358b56186f943c2bbd350c9005b9359133d47e954cfc561385319ae5bb8d7, 1198C0117593497A5EC5C199286AF1F9897469DC AFD8691FDAEDF03BDF6E460563F15A9B715376CA,
8.3.9, bf4d7b8ea60a356064f88485278bd6f941a230ec16f0fc401574ce1445ad6c77, 1198C0117593497A5EC5C199286AF1F9897469DC AFD8691FDAEDF03BDF6E460563F15A9B715376CA,

Просмотреть файл

@ -59,6 +59,7 @@ if [ "$debianFlavor" == "stretch" ]; then
PIP_VERSION="20.2.3"
else
PIP_VERSION="21.2.4"
pythonSdkFileName=python-$debianFlavor-$PYTHON_VERSION.tar.gz
fi