msquic/scripts/RemoteTests.json

{
    "Remote": {
        "Exe": "secnetperf",
        "Arguments": ""
    },
    "FullMatrix": false,
    "Tests": [
        {
            "TestName": "ThroughputUp",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -upload:12000 -stats:1 -sstats:1"
            },
            "Variables": [
                {
                    "Name": "Encryption",
                    "Local": {
                        "On": "",
                        "Off": "-encrypt:0"
                    },
                    "Default": "On"
                },
                {
                    "Name": "SendBuffering",
                    "Local": {
                        "On": "-sendbuf:1",
                        "Off": ""
                    },
                    "Default": "Off"
                }
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-10.0"
        },
        {
            "TestName": "TcpThroughputUp",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -upload:12000 -tcp:1"
            },
            "Variables": [
            ],
            "SkipKernel": true,
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "FailureDefault": "Result: 0 bytes @ 0 kbps (0.0 ms).",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-50.0"
        },
        {
            "TestName": "ThroughputUp",
            "Local" : {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -uni:1 -timed:1 -upload:12000 -stats:1 -sstats:1"
            },
            "Variables": [
                {
                    "Name": "Encryption",
                    "Local": {
                        "On": "",
                        "Off": "-encrypt:0"
                    },
                    "Default": "On"
                },
                {
                    "Name": "SendBuffering",
                    "Local": {
                        "On": "-sendbuf:1",
                        "Off": ""
                    },
                    "Default": "Off"
                }
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-60.0"
        },
        {
            "TestName": "ThroughputDown",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -download:12000 -stats:1"
            },
            "Variables": [
                {
                    "Name": "Encryption",
                    "Local": {
                        "On": "",
                        "Off": "-encrypt:0"
                    },
                    "Default": "On"
                }
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-10.0"
        },
        {
        "TestName": "TcpThroughputDown",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -download:12000 -tcp:1"
            },
            "Variables": [
            ],
            "SkipKernel": true,
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "FailureDefault": "Result: 0 bytes @ 0 kbps (0.0 ms).",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-50.0"
        },
        {
            "TestName": "ThroughputDown",
            "Local" : {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -uni:1 -timed:1 -download:12000 -stats:1"
            },
            "Variables": [
                {
                    "Name": "Encryption",
                    "Local": {
                        "On": "",
                        "Off": "-encrypt:0"
                    },
                    "Default": "On"
                }
            ],
            "AllowLoopback": true,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": ".*@ (.*) kbps.*",
            "Formats": ["{0} kbps"],
            "RegressionThreshold": "-60.0"
        },
        {
            "TestName": "RPS",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:lowlat -test:RPS -target:$RemoteAddress"
            },
            "Variables": [
                {
                    "Name": "ConnectionCount",
                    "Local": {
                        "1": "-conns:1 -requests:1 -runtime:30000",
                        "40": "-conns:40 -requests:20",
                        "250": "-conns:250 -requests:7500",
                        "1000": "-conns:1 -requests:1000"
                    },
                    "Default": "250"
                },
                {
                    "Name": "RequestSize",
                    "Local": {
                        "0": "-request:0"
                    },
                    "Default": "0"
                },
                {
                    "Name": "ResponseSize",
                    "Local": {
                        "0": "-response:0",
                        "512": "-response:512",
                        "4096": "-response:4096",
                        "16384": "-response:16384"
                    },
                    "Default": "4096"
                }
            ],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) RPS, Min: (.*), Max: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), StdErr: (.*)",
            "Formats": ["{0} RPS", "Minimum: {0}", "Maximum: {0}", "Percentiles: 50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Standard Error: {0}"],
            "RegressionThreshold": "-50.0"
        },
        {
            "TestName": "RPS",
            "Local": {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:lowlat -test:RPS -target:$RemoteAddress"
            },
            "Variables": [
                {
                    "Name": "ConnectionCount",
                    "Local": {
                        "1": "-conns:1 -requests:1",
                        "40": "-conns:40 -requests:20",
                        "250": "-conns:250 -requests:7500",
                        "1000": "-conns:1 -requests:1000"
                    },
                    "Default": "250"
                },
                {
                    "Name": "RequestSize",
                    "Local": {
                        "0": "-request:0"
                    },
                    "Default": "0"
                },
                {
                    "Name": "ResponseSize",
                    "Local": {
                        "0": "-response:0",
                        "512": "-response:512",
                        "4096": "-response:4096",
                        "16384": "-response:16384"
                    },
                    "Default": "4096"
                }
            ],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) RPS, Min: (.*), Max: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), StdErr: (.*)",
            "Formats": ["{0} RPS", "Minimum: {0}", "Maximum: {0}", "Percentiles: 50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Standard Error: {0}"],
            "RegressionThreshold": "-75.0"
        },
        {
            "TestName": "HPS",
            "Local": {
                "Platform": "Windows",
                "Tls": ["schannel", "openssl", "openssl3"],
                "Arch": ["x64", "x86", "arm", "arm64"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -test:HPS -target:$RemoteAddress -incrementtarget:1"
            },
            "Variables": [],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) HPS.*",
            "Formats": ["{0} HPS"],
            "RegressionThreshold": "-40.0"
        },
        {
            "TestName": "HPS",
            "Local": {
                "Platform": "linux",
                "Tls": ["openssl", "openssl3"],
                "Arch": ["x64", "arm"],
                "Exe": "secnetperf",
                "Arguments": "-exec:maxtput -test:HPS -target:$RemoteAddress"
            },
            "Variables": [],
            "AllowLoopback": false,
            "Iterations": 5,
            "RemoteReadyMatcher": "Started!",
            "ResultsMatcher": "Result: (.*) HPS.*",
            "Formats": ["{0} HPS"],
            "RegressionThreshold": "-75.0"
        }
    ]
}