Fix formatting of IPv6 addresses (#4339)

This commit is contained in:
Eddy Ashton 2022-10-18 09:20:34 +01:00 коммит произвёл GitHub
Родитель 74d3b744da
Коммит 1dd0f69b45
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
13 изменённых файлов: 378 добавлений и 37 удалений

Просмотреть файл

@ -294,6 +294,7 @@ if(BUILD_TESTS)
${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/thread_messaging.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/lru.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/hex.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/nonstd.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/contiguous_set.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/unit_strings.cpp
)

Просмотреть файл

@ -115,6 +115,7 @@ namespace nonstd
}
result.push_back(s.substr(separator_end));
return result;
}
@ -134,6 +135,63 @@ namespace nonstd
return std::make_tuple(v[0], v[1]);
}
/** Similar to split, but splits first from the end rather than the beginning.
* This means the results are returned in reverse order, and if max_split is
* specified then only the final N entries will be kept.
* split("A:B:C", ":", 1) => ["A", "B:C"]
* rsplit("A:B:C", ":", 1) => ["C", "A:B"]
*/
static inline std::vector<std::string_view> rsplit(
const std::string_view& s,
const std::string_view& separator = " ",
size_t max_split = SIZE_MAX)
{
std::vector<std::string_view> result;
auto prev_separator_start = s.size();
auto next_separator_start = s.rfind(separator);
while (next_separator_start != std::string_view::npos &&
result.size() < max_split)
{
auto separator_end = next_separator_start + separator.size();
result.push_back(
s.substr(separator_end, prev_separator_start - separator_end));
prev_separator_start = next_separator_start;
if (next_separator_start == 0)
{
break;
}
else
{
next_separator_start = s.rfind(separator, prev_separator_start - 1);
}
}
result.push_back(s.substr(0, prev_separator_start));
return result;
}
/* rsplit_1 wraps rsplit _and reverses the result order_ and allows writing
* things like:
* auto [host, port] = nonstd::rsplit_1("[1:2:3:4]:8000", ":")
*/
static inline std::tuple<std::string_view, std::string_view> rsplit_1(
const std::string_view& s, const std::string_view& separator)
{
const auto v = rsplit(s, separator, 1);
if (v.size() == 1)
{
// If separator is not present, return {"", s};
return std::make_tuple("", v[0]);
}
return std::make_tuple(v[1], v[0]);
}
/** These convert strings to upper or lower case, in-place
*/
static inline void to_upper(std::string& s)

Просмотреть файл

@ -169,7 +169,7 @@ namespace ccf
inline static std::pair<std::string, std::string> split_net_address(
const NodeInfoNetwork::NetAddress& addr)
{
auto [host, port] = nonstd::split_1(addr, ":");
auto [host, port] = nonstd::rsplit_1(addr, ":");
return std::make_pair(std::string(host), std::string(port));
}

261
src/ds/test/nonstd.cpp Normal file
Просмотреть файл

@ -0,0 +1,261 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the Apache 2.0 License.
#include "ccf/ds/nonstd.h"
#include <algorithm>
#include <doctest/doctest.h>
#include <string>
TEST_CASE("split" * doctest::test_suite("nonstd"))
{
{
INFO("Basic splits");
const auto s = "Good afternoon, good evening, and good night!";
{
INFO("Split by spaces");
auto v = nonstd::split(s, " ");
REQUIRE(v.size() == 7);
REQUIRE(v[0] == "Good");
REQUIRE(v[1] == "afternoon,");
REQUIRE(v[2] == "good");
REQUIRE(v[3] == "evening,");
REQUIRE(v[4] == "and");
REQUIRE(v[5] == "good");
REQUIRE(v[6] == "night!");
}
{
INFO("Split by commas");
auto v = nonstd::split(s, ",");
REQUIRE(v.size() == 3);
REQUIRE(v[0] == "Good afternoon");
REQUIRE(v[1] == " good evening");
REQUIRE(v[2] == " and good night!");
}
{
INFO("Split by comma-with-space");
auto v = nonstd::split(s, ", ");
REQUIRE(v.size() == 3);
REQUIRE(v[0] == "Good afternoon");
REQUIRE(v[1] == "good evening");
REQUIRE(v[2] == "and good night!");
{
INFO("split(max_splits=3)");
{
auto v = nonstd::split(s, " ", 3);
// NB: max_splits=3 => 4 returned segments
REQUIRE(v.size() == 4);
REQUIRE(v[0] == "Good");
REQUIRE(v[1] == "afternoon,");
REQUIRE(v[2] == "good");
REQUIRE(v[3] == "evening, and good night!");
}
{
auto v = nonstd::split(s, "afternoon", 3);
// NB: max_splits=3, but only 1 split possible => 2 returned segments
REQUIRE(v.size() == 2);
REQUIRE(v[0] == "Good ");
REQUIRE(v[1] == ", good evening, and good night!");
}
}
{
INFO("split_1");
auto t = nonstd::split_1(s, ", ");
REQUIRE(std::get<0>(t) == "Good afternoon");
REQUIRE(std::get<1>(t) == "good evening, and good night!");
}
}
{
INFO("Split by commas");
auto v = nonstd::split(s, ",");
REQUIRE(v.size() == 3);
REQUIRE(v[0] == "Good afternoon");
REQUIRE(v[1] == " good evening");
REQUIRE(v[2] == " and good night!");
}
{
INFO("Split by 'oo'");
auto v = nonstd::split(s, "oo");
REQUIRE(v.size() == 5);
REQUIRE(v[0] == "G");
REQUIRE(v[1] == "d aftern");
REQUIRE(v[2] == "n, g");
REQUIRE(v[3] == "d evening, and g");
REQUIRE(v[4] == "d night!");
}
}
{
INFO("Edge cases");
{
const auto s = " bob ";
auto v = nonstd::split(s, " ");
REQUIRE(v.size() == 4);
REQUIRE(v[0].empty());
REQUIRE(v[1].empty());
REQUIRE(v[2] == "bob");
REQUIRE(v[3].empty());
}
{
const auto s = "bobbob";
{
auto v = nonstd::split(s, " ");
REQUIRE(v.size() == 1);
REQUIRE(v[0] == "bobbob");
}
{
auto v = nonstd::split(s, "bob");
REQUIRE(v.size() == 3);
REQUIRE(v[0].empty());
REQUIRE(v[1].empty());
REQUIRE(v[2].empty());
}
{
auto t = nonstd::split_1(s, "bob");
REQUIRE(std::get<0>(t).empty());
REQUIRE(std::get<1>(t) == "bob");
}
}
{
const auto s = "";
{
auto v = nonstd::split(s, " ");
REQUIRE(v.size() == 1);
REQUIRE(v[0].empty());
}
{
auto v = nonstd::split(s, "bob");
REQUIRE(v.size() == 1);
REQUIRE(v[0].empty());
}
{
auto t = nonstd::split_1(s, " ");
REQUIRE(std::get<0>(t).empty());
REQUIRE(std::get<1>(t).empty());
}
}
}
}
TEST_CASE("rsplit" * doctest::test_suite("nonstd"))
{
{
INFO("Basic rsplits");
const auto s = "Good afternoon, good evening, and good night!";
{
INFO("rsplit by spaces");
auto v = nonstd::rsplit(s, " ");
REQUIRE(v.size() == 7);
REQUIRE(v[0] == "night!");
REQUIRE(v[1] == "good");
REQUIRE(v[2] == "and");
REQUIRE(v[3] == "evening,");
REQUIRE(v[4] == "good");
REQUIRE(v[5] == "afternoon,");
REQUIRE(v[6] == "Good");
}
{
INFO("rsplit(max_splits=3)");
{
auto v = nonstd::rsplit(s, " ", 3);
// NB: max_splits=3 => 4 returned segments
REQUIRE(v.size() == 4);
REQUIRE(v[0] == "night!");
REQUIRE(v[1] == "good");
REQUIRE(v[2] == "and");
REQUIRE(v[3] == "Good afternoon, good evening,");
}
{
auto v = nonstd::rsplit(s, "afternoon", 3);
// NB: max_splits=3, but only 1 split possible => 2 returned segments
REQUIRE(v.size() == 2);
REQUIRE(v[0] == ", good evening, and good night!");
REQUIRE(v[1] == "Good ");
}
}
{
INFO("rsplit_1");
auto t = nonstd::rsplit_1(s, ", ");
REQUIRE(std::get<0>(t) == "Good afternoon, good evening");
REQUIRE(std::get<1>(t) == "and good night!");
}
}
{
INFO("Edge cases");
{
const auto s = " bob ";
auto v = nonstd::rsplit(s, " ");
REQUIRE(v.size() == 4);
REQUIRE(v[0].empty());
REQUIRE(v[1] == "bob");
REQUIRE(v[2].empty());
REQUIRE(v[3].empty());
}
{
const auto s = "bobbob";
{
auto v = nonstd::rsplit(s, " ");
REQUIRE(v.size() == 1);
REQUIRE(v[0] == "bobbob");
}
{
auto v = nonstd::rsplit(s, "bob");
REQUIRE(v.size() == 3);
REQUIRE(v[0].empty());
REQUIRE(v[1].empty());
REQUIRE(v[2].empty());
}
{
auto t = nonstd::rsplit_1(s, "bob");
REQUIRE(std::get<0>(t) == "bob");
REQUIRE(std::get<1>(t).empty());
}
}
{
const auto s = "";
{
auto v = nonstd::rsplit(s, " ");
REQUIRE(v.size() == 1);
REQUIRE(v[0].empty());
}
{
auto v = nonstd::rsplit(s, "bob");
REQUIRE(v.size() == 1);
REQUIRE(v[0].empty());
}
{
auto t = nonstd::rsplit_1(s, " ");
REQUIRE(std::get<0>(t).empty());
REQUIRE(std::get<1>(t).empty());
}
}
}
}

Просмотреть файл

@ -17,7 +17,7 @@ namespace asynchost
{
public:
static bool resolve(
const std::string& host,
const std::string& host_,
const std::string& service,
void* ud,
uv_getaddrinfo_cb cb,
@ -32,6 +32,11 @@ namespace asynchost
auto resolver = new uv_getaddrinfo_t;
resolver->data = ud;
std::string host =
(host_.starts_with("[") && host_.ends_with("]") ?
host_.substr(1, host_.size() - 2) :
host_);
int rc;
if (async)

Просмотреть файл

@ -77,7 +77,7 @@ namespace asynchost
std::pair<std::string, std::string> addr_to_str(
const sockaddr* addr, int address_family = AF_INET)
{
constexpr auto buf_len = UV_IF_NAMESIZE;
constexpr auto buf_len = INET6_ADDRSTRLEN;
char buf[buf_len] = {};
int rc;
@ -89,7 +89,8 @@ namespace asynchost
LOG_FAIL_FMT("uv_ip6_name failed: {}", uv_strerror(rc));
}
return {buf, fmt::format("{}", ntohs(in6->sin6_port))};
return {
fmt::format("[{}]", buf), fmt::format("{}", ntohs(in6->sin6_port))};
}
assert(address_family == AF_INET);

Просмотреть файл

@ -1682,7 +1682,7 @@ namespace ccf
{
// IP address components are purely numeric. DNS names may be largely
// numeric, but at least the final component (TLD) must not be
// all-numeric. So this distinguishes "1.2.3.4" (and IP address) from
// all-numeric. So this distinguishes "1.2.3.4" (an IP address) from
// "1.2.3.c4m" (a DNS name). "1.2.3." is invalid for either, and will
// throw. Attempts to handle IPv6 by also splitting on ':', but this is
// untested.
@ -1691,8 +1691,7 @@ namespace ccf
if (final_component.empty())
{
throw std::runtime_error(fmt::format(
"{} has a trailing period, is not a valid hostname",
final_component));
"{} has a trailing period, is not a valid hostname", hostname));
}
for (const auto c : final_component)
{

Просмотреть файл

@ -299,10 +299,9 @@ class CurlClient:
"""
def __init__(
self, host, port, ca=None, session_auth=None, signing_auth=None, **kwargs
self, hostname, ca=None, session_auth=None, signing_auth=None, **kwargs
):
self.host = host
self.port = port
self.hostname = hostname
self.ca = ca
self.session_auth = session_auth
self.signing_auth = signing_auth
@ -323,7 +322,7 @@ class CurlClient:
else:
cmd = ["curl"]
url = f"{self.protocol}://{self.host}:{self.port}{request.path}"
url = f"{self.protocol}://{self.hostname}{request.path}"
cmd += [url, "-X", request.http_verb, "-i", f"-m {timeout}"]
@ -404,16 +403,14 @@ class RequestClient:
def __init__(
self,
host: str,
port: int,
hostname: str,
ca: str,
session_auth: Optional[Identity] = None,
signing_auth: Optional[Identity] = None,
common_headers: Optional[dict] = None,
**kwargs,
):
self.host = host
self.port = port
self.hostname = hostname
self.ca = ca
self.session_auth = session_auth
self.signing_auth = signing_auth
@ -491,7 +488,7 @@ class RequestClient:
try:
response = self.session.request(
request.http_verb,
url=f"{self.protocol}://{self.host}:{self.port}{request.path}",
url=f"{self.protocol}://{self.hostname}{request.path}",
auth=auth,
headers=extra_headers,
follow_redirects=request.allow_redirects,
@ -551,7 +548,7 @@ class CCFClient:
**kwargs,
):
self.connection_timeout = connection_timeout
self.hostname = f"{host}:{port}"
self.hostname = infra.interfaces.make_address(host, port)
self.name = f"[{self.hostname}]"
self.description = description or self.name
self.is_connected = False
@ -560,11 +557,11 @@ class CCFClient:
if curl or os.getenv("CURL_CLIENT"):
self.client_impl = CurlClient(
host, port, ca, session_auth, signing_auth, **kwargs
self.hostname, ca, session_auth, signing_auth, **kwargs
)
else:
self.client_impl = RequestClient(
host, port, ca, session_auth, signing_auth, common_headers, **kwargs
self.hostname, ca, session_auth, signing_auth, common_headers, **kwargs
)
def _response(self, response: Response) -> Response:

Просмотреть файл

@ -4,15 +4,22 @@
from dataclasses import dataclass
from typing import Optional, Dict
from enum import Enum, auto
import urllib.parse
from loguru import logger as LOG
def split_address(addr, default_port=0):
host, *port = addr.split(":")
return host, (int(port[0]) if port else default_port)
def split_netloc(netloc, default_port=0):
url = f"http://{netloc}"
parsed = urllib.parse.urlparse(url)
return parsed.hostname, (parsed.port if parsed.port else default_port)
def make_address(host, port=0):
return f"{host}:{port}"
if ":" in host:
return f"[{host}]:{port}"
else:
return f"{host}:{port}"
DEFAULT_TRANSPORT_PROTOCOL = "tcp"
@ -91,7 +98,7 @@ class RPCInterface(Interface):
@staticmethod
def to_json(interface):
r = {
"bind_address": f"{interface.host}:{interface.port}",
"bind_address": make_address(interface.host, interface.port),
"protocol": f"{interface.transport}",
"app_protocol": interface.app_protocol.name,
"published_address": f"{interface.public_host}:{interface.public_port or 0}",
@ -112,10 +119,13 @@ class RPCInterface(Interface):
def from_json(json):
interface = RPCInterface()
interface.transport = json.get("protocol", DEFAULT_TRANSPORT_PROTOCOL)
interface.host, interface.port = split_address(json.get("bind_address"))
interface.host, interface.port = split_netloc(json.get("bind_address"))
LOG.warning(
f"Converted {json.get('bind_address')} to {interface.host} and {interface.port}"
)
published_address = json.get("published_address")
if published_address is not None:
interface.public_host, interface.public_port = split_address(
interface.public_host, interface.public_port = split_netloc(
published_address
)
interface.max_open_sessions_soft = json.get(
@ -172,8 +182,8 @@ class HostSpec:
pub_host, pub_port = None, None
if "," in address:
address, published_address = address.split(",")
pub_host, pub_port = split_address(published_address)
host, port = split_address(address)
pub_host, pub_port = split_netloc(published_address)
host, port = split_netloc(address)
return HostSpec(
rpc_interfaces={
PRIMARY_RPC_INTERFACE: RPCInterface(

Просмотреть файл

@ -4,6 +4,7 @@ import re
import socket
from random import randrange as rr
from subprocess import check_output
from os import getenv
def ephemeral_range():
@ -79,4 +80,8 @@ def two_different(finder, *args, **kwargs):
def expand_localhost():
return ".".join((str(b) for b in (127, rr(1, 255), rr(1, 255), rr(2, 255))))
ipv4 = ".".join((str(b) for b in (127, rr(1, 255), rr(1, 255), rr(2, 255))))
if getenv("CCF_IPV6"):
return f"::ffff:{ipv4}"
else:
return ipv4

Просмотреть файл

@ -287,9 +287,7 @@ class Network:
workspace=args.workspace,
label=args.label,
common_dir=self.common_dir,
target_rpc_address=infra.interfaces.make_address(
target_node.get_public_rpc_host(), target_node.get_public_rpc_port()
),
target_rpc_address=target_node.get_public_rpc_address(),
snapshots_dir=snapshots_dir,
read_only_snapshots_dir=read_only_snapshots_dir,
ledger_dir=current_ledger_dir,

Просмотреть файл

@ -358,7 +358,7 @@ class Node:
addresses = json.load(f)
for interface_name, resolved_address in addresses.items():
host, port = infra.interfaces.split_address(resolved_address)
host, port = infra.interfaces.split_netloc(resolved_address)
interface = interfaces[interface_name]
if self.remote_shim != infra.remote_shim.DockerShim:
assert (
@ -562,6 +562,14 @@ class Node:
):
return self.host.rpc_interfaces[interface_name].public_port
def get_public_rpc_address(
self, interface_name=infra.interfaces.PRIMARY_RPC_INTERFACE
):
interface = self.host.rpc_interfaces[interface_name]
return infra.interfaces.make_address(
interface.public_host, interface.public_port
)
def retrieve_self_signed_cert(self, *args, **kwargs):
# Retrieve and overwrite node self-signed certificate in common directory
with self.client(*args, **kwargs) as c:

Просмотреть файл

@ -11,7 +11,6 @@ import os
import shutil
from loguru import logger as LOG
DEFAULT_NODES = ["local://127.0.0.1:8000"]
@ -106,10 +105,9 @@ def run(args):
LOG.info("Started CCF network with the following nodes:")
for node in nodes:
LOG.info(
" Node [{}] = https://{}:{}".format(
" Node [{}] = https://{}".format(
pad_node_id(node.local_node_id),
node.get_public_rpc_host(),
node.get_public_rpc_port(),
node.get_public_rpc_address(),
)
)