msquic/scripts/RemoteTests.json

{
    "Remote": {
        "Exe": "secnetperf",
        "Arguments": ""
    },
    "FullMatrix": false,
    "Tests": [
        {
            "TestName": "ThroughputUp",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12s -ptput:1 -tcp:0"
            },
            "Variables": [
                {
                    "Name": "Encryption",
                    "Local": {
                        "On": "",
                        "Off": "-encrypt:0"
                    },
                    "Default": "On"
                },
                {
                    "Name": "SendBuffering",
                    "Local": {
                        "On": "-sendbuf:1",
                        "Off": ""
                    },
                    "Default": "Off"
                }
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-10.0"
        },
        {
            "TestName": "TcpThroughputUp",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12s -ptput:1 -tcp:1"
            },
            "Variables": [
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "FailureDefault": "Result: 0 bytes @ 0 kbps (0.0 ms).",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-50.0"
        },
        {
            "TestName": "ThroughputUp",
            "Local" : {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12s -ptput:1 -tcp:0"
            },
            "Variables": [
                {
                    "Name": "Encryption",
                    "Local": {
                        "On": "",
                        "Off": "-encrypt:0"
                    },
                    "Default": "On"
                },
                {
                    "Name": "SendBuffering",
                    "Local": {
                        "On": "-sendbuf:1",
                        "Off": ""
                    },
                    "Default": "Off"
                }
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-60.0"
        },
        {
            "TestName": "TcpThroughputUp",
            "Local" : {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12s -ptput:1 -tcp:1"
            },
            "Variables": [
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "FailureDefault": "Result: 0 bytes @ 0 kbps (0.0 ms).",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-60.0"
        },
        {
            "TestName": "ThroughputDown",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12s -ptput:1 -pconn:1 -tcp:0"
            },
            "Variables": [
                {
                    "Name": "Encryption",
                    "Local": {
                        "On": "",
                        "Off": "-encrypt:0"
                    },
                    "Default": "On"
                }
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-10.0"
        },
        {
        "TestName": "TcpThroughputDown",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12s -ptput:1 -pconn:1 -tcp:1"
            },
            "Variables": [
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "FailureDefault": "Result: 0 bytes @ 0 kbps (0.0 ms).",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-50.0"
        },
        {
            "TestName": "ThroughputDown",
            "Local" : {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12s -ptput:1 -pconn:1 -tcp:0"
            },
            "Variables": [
                {
                    "Name": "Encryption",
                    "Local": {
                        "On": "",
                        "Off": "-encrypt:0"
                    },
                    "Default": "On"
                }
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-60.0"
        },
        {
            "TestName": "TcpThroughputDown",
            "Local" : {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12s -ptput:1 -pconn:1 -tcp:1"
            },
            "Variables": [
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "FailureDefault": "Result: 0 bytes @ 0 kbps (0.0 ms).",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-60.0"
        },
        {
            "TestName": "RPS",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:0"
            },
            "Variables": [
                {
                    "Name": "ConnectionCount",
                    "Local": {
                        "1": "-conns:1 -requests:1 -run:30s",
                        "40": "-conns:40 -requests:1 -run:10s",
                        "250": "-conns:250 -requests:30 -run:10s",
                        "1000": "-conns:1 -requests:1000 -run:10s"
                    },
                    "Default": "250"
                },
                {
                    "Name": "RequestSize",
                    "Local": {
                        "0": "-request:0"
                    },
                    "Default": "0"
                },
                {
                    "Name": "ResponseSize",
                    "Local": {
                        "0": "-response:0",
                        "512": "-response:512",
                        "4096": "-response:4096",
                        "16384": "-response:16384"
                    },
                    "Default": "4096"
                }
            ],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
            "Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
            "RegressionThreshold": "-50.0"
        },
        {
            "TestName": "TcpRPS",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:1"
            },
            "Variables": [
                {
                    "Name": "ConnectionCount",
                    "Local": {
                        "1": "-conns:1 -requests:1 -run:30s",
                        "40": "-conns:40 -requests:1 -run:10s",
                        "250": "-conns:250 -requests:30 -run:10s",
                        "1000": "-conns:1 -requests:1000 -run:10s"
                    },
                    "Default": "250"
                },
                {
                    "Name": "RequestSize",
                    "Local": {
                        "0": "-request:0"
                    },
                    "Default": "0"
                },
                {
                    "Name": "ResponseSize",
                    "Local": {
                        "0": "-response:0",
                        "512": "-response:512",
                        "4096": "-response:4096",
                        "16384": "-response:16384"
                    },
                    "Default": "4096"
                }
            ],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
            "Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
            "RegressionThreshold": "-50.0"
        },
        {
            "TestName": "RPS",
            "Local": {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:0"
            },
            "Variables": [
                {
                    "Name": "ConnectionCount",
                    "Local": {
                        "1": "-conns:1 -requests:1 -run:30s",
                        "40": "-conns:40 -requests:1 -run:10s",
                        "250": "-conns:250 -requests:30 -run:10s",
                        "1000": "-conns:1 -requests:1000 -run:10s"
                    },
                    "Default": "250"
                },
                {
                    "Name": "RequestSize",
                    "Local": {
                        "0": "-request:0"
                    },
                    "Default": "0"
                },
                {
                    "Name": "ResponseSize",
                    "Local": {
                        "0": "-response:0",
                        "512": "-response:512",
                        "4096": "-response:4096",
                        "16384": "-response:16384"
                    },
                    "Default": "4096"
                }
            ],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
            "Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
            "RegressionThreshold": "-75.0"
        },
        {
            "TestName": "TcpRPS",
            "Local": {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:1"
            },
            "Variables": [
                {
                    "Name": "ConnectionCount",
                    "Local": {
                        "1": "-conns:1 -requests:1 -run:30s",
                        "40": "-conns:40 -requests:1 -run:10s",
                        "250": "-conns:250 -requests:30 -run:10s",
                        "1000": "-conns:1 -requests:1000 -run:10s"
                    },
                    "Default": "250"
                },
                {
                    "Name": "RequestSize",
                    "Local": {
                        "0": "-request:0"
                    },
                    "Default": "0"
                },
                {
                    "Name": "ResponseSize",
                    "Local": {
                        "0": "-response:0",
                        "512": "-response:512",
                        "4096": "-response:4096",
                        "16384": "-response:16384"
                    },
                    "Default": "4096"
                }
            ],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
            "Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
            "RegressionThreshold": "-75.0"
        },
        {
            "TestName": "HPS",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -rconn:1 -conns:800 -share:1 -run:10s -tcp:0 -prate:1"
            },
            "Variables": [],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) HPS.*",
            "Formats": ["{0} HPS"],
            "RegressionThreshold": "-40.0"
        },
        {
            "TestName": "TcpHPS",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -rconn:1 -conns:800 -share:1 -run:10s -tcp:1 -prate:1"
            },
            "Variables": [],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) HPS.*",
            "Formats": ["{0} HPS"],
            "RegressionThreshold": "-40.0"
        },
        {
            "TestName": "HPS",
            "Local": {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -rconn:1 -conns:800 -share:1 -run:10s -tcp:0 -prate:1"
            },
            "Variables": [],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) HPS.*",
            "Formats": ["{0} HPS"],
            "RegressionThreshold": "-75.0"
        },
        {
            "TestName": "TcpHPS",
            "Local": {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -rconn:1 -conns:800 -share:1 -run:10s -tcp:1 -prate:1"
            },
            "Variables": [],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) HPS.*",
            "Formats": ["{0} HPS"],
            "RegressionThreshold": "-75.0"
        }
    ]
}