Bug 1738708 - Update to Botan 2.18.2. r=rjl

Differential Revision: https://phabricator.services.mozilla.com/D130099

third_party/botan/configure.py

@@ -333,6 +333,9 @@ def process_command_line(args): # pylint: disable=too-many-locals,too-many-state
     target_group.add_option('--ldflags', metavar='FLAGS',
                             help='set linker flags', default=None)
 
+    target_group.add_option('--extra-libs', metavar='LIBS',
+                            help='specify extra libraries to link against', default='')
+
     target_group.add_option('--ar-command', dest='ar_command', metavar='AR', default=None,
                             help='set path to static archive creator')
 
@@ -2007,6 +2010,12 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
         else:
             return '%s %s' % (options.compiler_cache, cxx)
 
+    def extra_libs(libs, cc):
+        if libs is None:
+            return ''
+
+        return ' '.join([(cc.add_lib_option % lib) for lib in libs.split(',') if lib != ''])
+
     variables = {
         'version_major':  Version.major(),
         'version_minor':  Version.minor(),
@@ -2027,6 +2036,12 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
         'all_targets': ' '.join(all_targets(options)),
         'install_targets': ' '.join(install_targets(options)),
 
+        'public_headers': sorted([os.path.basename(h) for h in build_paths.public_headers]),
+        'internal_headers': sorted([os.path.basename(h) for h in build_paths.internal_headers]),
+        'external_headers':  sorted([os.path.basename(h) for h in build_paths.external_headers]),
+
+        'abs_root_dir': os.path.dirname(os.path.realpath(__file__)),
+
         'base_dir': source_paths.base_dir,
         'src_dir': source_paths.src_dir,
         'test_data_dir': source_paths.test_data_dir,
@@ -2095,6 +2110,7 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
 
         'os': options.os,
         'arch': options.arch,
+        'compiler': options.compiler,
         'cpu_family': arch.family,
         'endian': options.with_endian,
         'cpu_is_64bit': arch.wordsize == 64,
@@ -2131,6 +2147,7 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
         'cc_sysroot': sysroot_option(),
         'cc_compile_flags': options.cxxflags or cc.cc_compile_flags(options),
         'ldflags': options.ldflags or '',
+        'extra_libs': extra_libs(options.extra_libs, cc),
         'cc_warning_flags': cc.cc_warning_flags(options),
         'output_to_exe': cc.output_to_exe,
         'cc_macro': cc.macro_name,
@@ -2181,6 +2198,11 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
         'mod_list': sorted([m.basename for m in modules])
     }
 
+    variables['installed_include_dir'] = os.path.join(
+        variables['prefix'],
+        variables['includedir'],
+        'botan-%d' % (Version.major()), 'botan')
+
     if cc.basename == 'msvc' and variables['cxx_abi_flags'] != '':
         # MSVC linker doesn't support/need the ABI options,
         # just transfer them over to just the compiler invocations
@@ -3006,6 +3028,16 @@ def canonicalize_options(options, info_os, info_arch):
     if options.build_fuzzers == 'libfuzzer' and options.fuzzer_lib is None:
         options.fuzzer_lib = 'Fuzzer'
 
+    if options.ldflags is not None:
+        extra_libs = []
+        link_to_lib = re.compile('^-l(.*)')
+        for flag in options.ldflags.split(' '):
+            match = link_to_lib.match(flag)
+            if match:
+                extra_libs.append(match.group(1))
+
+        options.extra_libs += ','.join(extra_libs)
+
 # Checks user options for consistency
 # This method DOES NOT change options on behalf of the user but explains
 # why the given configuration does not work.

third_party/botan/news.rst

@@ -1,11 +1,81 @@
 Release Notes
 ========================================
 
+Version 2.18.2, 2021-10-25
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* Avoid using short exponents when encrypting in ElGamal, as some PGP
+  implementations generate keys with parameters that are weak when
+  short exponents are used (GH #2794)
+
+* Fix a low risk OAEP decryption side channel (GH #2797)
+
+* Work around a miscompilation of SHA-3 caused by a bug in Clang 12
+  and XCode 13. (GH #2826)
+
+* Remove support in OpenSSL provider for algorithms which are
+  disabled by default in OpenSSL 3.0 (GH #2823, #2814)
+
+* Add CI based on GitHub actions to replace Travis CI (GH #2632)
+
+* Fix the online OCSP test, as the certificate involved had expired.
+  (GH #2799)
+
+* Fix some test failures induced by the expiration of the trust root
+  "DST Root CA X3" (GH #2820)
+
+Version 2.18.1, 2021-05-09
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* Fix a build regression in 2.18.0 which caused linker flags which
+  contain ``-l`` within them (such as ``-fuse-linker-plugin``) to
+  be misinterpreted. (GH #2715)
+
+* Fix a bug which caused decoding a certificate which contained
+  more than one name in a single RDN. (GH #2611 #2630 #2724)
+
+* Fix a bug which caused OID lookup failures when run in a locale
+  which uses thousands separators (pt_BR was reported as having
+  this issue). (GH #2732 #2730 #2237)
+
+* DNS names in name constraints were compared with case sensitivity, which
+  could cause valid certificates to be rejected. (GH #2739 #2735)
+
+* X.509 name constraint extensions were rejected if non-critical. RFC 5280
+  requires conforming CAs issue such extensions as critical, but not all
+  certificates are compliant, and all other known implementations do not
+  require this. (GH #2739 #2736)
+
+* X.509 name constraints were incorrectly applied to the certificate which
+  included the constraint. (GH #2739 #2737)
+
+Version 2.18.0, 2021-04-15
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* Add support for implementing custom RNG objects through the
+  FFI interface (GH #2627 #2600)
+
+* Improve safegcd bounds, improving runtime performance (GH #2628 #2619)
+
+* Fix a bug introduced in 2.9.0 where BigInt::operator< would return
+  an incorrect result if both operands were negative. (GH #2641 #2638)
+
+* Reject non-TLS messages as quickly as possible without waiting for
+  a full record. (GH #2676)
+
+* Add build support for RISC-V 32
+
+* Fixes for TLS::Stream::async_shutdown (GH #2673)
+
+* Fix a regression introduced in 2.17.0 where LDFLAGS which add an extra
+  library (such as ``-latomic`` needed on SPARC) were not always applied
+  effectively. (GH #2622 #2623 #2625)
+
 Version 2.17.3, 2020-12-21
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-* Change base64, base58, base32, and hex encoding and decoding opearations
+* CVE-2021-24115 Change base64, base58, base32, and hex encoding and
-  to run in constant time (GH #2549)
+  decoding operations to run in constant time (GH #2549)
 
 * Fix a build problem on PPC64 building with Clang (GH #2547)
 
@@ -27,7 +97,7 @@ Version 2.17.2, 2020-11-13
 
 * Resolve an issue in the modular square root algorithm where a loop
   to find a quadratic non-residue could, for a carefully chosen
-  composite modulus, not terminte in a timely manner. (GH #2482 #2476)
+  composite modulus, not terminate in a timely manner. (GH #2482 #2476)
 
 * Fix a regression in MinGW builds introduced in 2.17.1
 

third_party/botan/readme.rst

@@ -27,9 +27,9 @@ If you think you have found a security issue, see the `security page
 <https://botan.randombit.net/security.html>`_ for contact information.
 
 The latest release is
-`2.17.3 <https://botan.randombit.net/releases/Botan-2.17.3.tar.xz>`_
+`2.18.2 <https://botan.randombit.net/releases/Botan-2.18.2.tar.xz>`_
-`(sig) <https://botan.randombit.net/releases/Botan-2.17.3.tar.xz.asc>`_,
+`(sig) <https://botan.randombit.net/releases/Botan-2.18.2.tar.xz.asc>`_,
-released on 2020-12-21.
+released on 2021-10-25.
 All releases are signed with a `PGP key <https://botan.randombit.net/pgpkey.txt>`_.
 See the `release notes <https://botan.randombit.net/news.html>`_ for
 what is new. Botan is also available through most

third_party/botan/src/bogo_shim/bogo_shim.cpp

@@ -110,6 +110,7 @@ std::string map_to_bogo_error(const std::string& e)
          { "Encoding error: Cannot encode PSS string, output length too small", ":NO_COMMON_SIGNATURE_ALGORITHMS:" },
          { "Expected TLS but got a record with DTLS version", ":WRONG_VERSION_NUMBER:" },
          { "Finished message didn't verify", ":DIGEST_CHECK_FAILED:" },
+         { "Got unexpected TLS record version", ":WRONG_VERSION_NUMBER:" },
          { "Inconsistent length in certificate request", ":DECODE_ERROR:" },
          { "Inconsistent values in fragmented DTLS handshake header", ":FRAGMENT_MISMATCH:" },
          { "Invalid CertificateRequest: Length field outside parameters", ":DECODE_ERROR:" },

third_party/botan/src/bogo_shim/config.json

@@ -23,6 +23,7 @@
         "ExportTrafficSecrets-*": "No TLS 1.3",
         "IgnoreClientVersionOrder": "No TLS 1.3",
         "Resume-Server-OmitPSKsOnSecondClientHello": "No TLS 1.3",
+        "Http*": "No support for HTTP detection",
 
         "DuplicateCertCompressionExt*": "No support for 1.3 cert compression extension",
 

third_party/botan/src/build-data/arch/arm32.txt

@@ -10,7 +10,7 @@ evbarm # For NetBSD
 
 armv7
 armv7l
-arvm7a
+armv7a
 armv7-a
 
 armv8l # For AlpineLinux

third_party/botan/src/build-data/arch/riscv32.txt

@@ -0,0 +1,2 @@
+family riscv
+endian little

third_party/botan/src/build-data/arch/riscv64.txt

@@ -1,2 +1,3 @@
 family riscv
 endian little
+wordsize 64

third_party/botan/src/build-data/makefile.in

@@ -21,7 +21,7 @@ LDFLAGS        = %{ldflags}
 EXE_LINK_CMD   = %{exe_link_cmd}
 
 LIB_LINKS_TO   = %{external_link_cmd} %{link_to}
-EXE_LINKS_TO   = %{link_to_botan} $(LIB_LINKS_TO)
+EXE_LINKS_TO   = %{link_to_botan} $(LIB_LINKS_TO) %{extra_libs}
 
 BUILD_FLAGS    = $(ABI_FLAGS) $(LANG_FLAGS) $(CXXFLAGS) $(WARN_FLAGS)
 

third_party/botan/src/build-data/version.txt

@@ -1,11 +1,11 @@
 
 release_major = 2
-release_minor = 17
+release_minor = 18
-release_patch = 3
+release_patch = 2
 release_suffix = ''
-release_so_abi_rev = 17
+release_so_abi_rev = 18
 
 # These are set by the distribution script
-release_vc_rev = 'git:bcda19704da482c57eb0bce786cebb97f378f146'
+release_vc_rev = 'git:a44f1489239e80937ca67564ff103421e5584069'
-release_datestamp = 20201221
+release_datestamp = 20211025
 release_type = 'release'

third_party/botan/src/fuzzer/bn_cmp.cpp

@@ -0,0 +1,74 @@
+/*
+* (C) 2021 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include "fuzzers.h"
+
+#include <botan/bigint.h>
+
+void fuzz(const uint8_t in[], size_t len)
+   {
+   const size_t max_bits = 512;
+
+   if(len < 3 || len > 1 + 2*(max_bits/8))
+      return;
+
+   const uint8_t signs = in[0];
+   const size_t x_len = (len - 1) / 2;
+
+   Botan::BigInt x = Botan::BigInt::decode(in + 1, x_len);
+   Botan::BigInt y = Botan::BigInt::decode(in + 1 + x_len, len - x_len - 1);
+
+   if(signs & 1)
+      x.flip_sign();
+   if(signs & 2)
+      y.flip_sign();
+
+   const Botan::BigInt d1 = x - y;
+   const Botan::BigInt d2 = y - x;
+
+   FUZZER_ASSERT_TRUE(d1.cmp(d2, false) == 0);
+
+   const bool is_eq = (x == y);
+   const bool is_lt = (x < y);
+   const bool is_gt = (x > y);
+   const bool is_lte = (x <= y);
+   const bool is_gte = (x >= y);
+
+   if(is_eq)
+      {
+      FUZZER_ASSERT_TRUE(d1.is_zero());
+      FUZZER_ASSERT_TRUE(d2.is_zero());
+      }
+
+   if(is_lte)
+      {
+      FUZZER_ASSERT_TRUE(is_lt || is_eq);
+      }
+
+   if(is_gte)
+      {
+      FUZZER_ASSERT_TRUE(is_gt || is_eq);
+      }
+
+   if(is_lt)
+      {
+      FUZZER_ASSERT_TRUE(!is_gt);
+      FUZZER_ASSERT_TRUE(d1.is_nonzero());
+      FUZZER_ASSERT_TRUE(d2.is_nonzero());
+      FUZZER_ASSERT_TRUE(d1.is_negative());
+      FUZZER_ASSERT_TRUE(d2.is_positive());
+      }
+
+   if(is_gt)
+      {
+      FUZZER_ASSERT_TRUE(!is_lt);
+      FUZZER_ASSERT_TRUE(d1.is_nonzero());
+      FUZZER_ASSERT_TRUE(d2.is_nonzero());
+      FUZZER_ASSERT_TRUE(d1.is_positive());
+      FUZZER_ASSERT_TRUE(d2.is_negative());
+      }
+   }
+

third_party/botan/src/lib/asn1/asn1_oid.cpp

@@ -98,6 +98,7 @@ OID::OID(const std::string& oid_str)
 std::string OID::to_string() const
    {
    std::ostringstream oss;
+   oss.imbue(std::locale("C"));
    for(size_t i = 0; i != m_id.size(); ++i)
       {
       oss << m_id[i];

third_party/botan/src/lib/ffi/ffi.cpp

@@ -200,6 +200,10 @@ uint32_t botan_ffi_api_version()
 
 int botan_ffi_supports_api(uint32_t api_version)
    {
+   // This is the API introduced in 2.18
+   if(api_version == 20210220)
+      return BOTAN_FFI_SUCCESS;
+
    // This is the API introduced in 2.13
    if(api_version == 20191214)
       return BOTAN_FFI_SUCCESS;

third_party/botan/src/lib/ffi/ffi.h

@@ -1,6 +1,7 @@
 /*
 * FFI (C89 API)
 * (C) 2015,2017 Jack Lloyd
+* (C) 2021 René Fischer
 *
 * Botan is released under the Simplified BSD License (see license.txt)
 */
@@ -200,6 +201,20 @@ typedef struct botan_rng_struct* botan_rng_t;
 */
 BOTAN_PUBLIC_API(2,0) int botan_rng_init(botan_rng_t* rng, const char* rng_type);
 
+/**
+* Initialize a custom random number generator from a set of callback functions
+* @param rng rng object
+* @param rng_name name of the rng
+* @param context An application-specific context passed to the callback functions
+* @param get_cb Callback for getting random bytes from the rng, return 0 for success
+* @param add_entry_cb Callback for adding entropy to the rng, return 0 for success, may be NULL
+* @param destroy_cb Callback called when rng is destroyed, may be NULL
+*/
+BOTAN_PUBLIC_API(2,18) int botan_rng_init_custom(botan_rng_t* rng_out, const char* rng_name, void* context,
+                                                 int(* get_cb)(void* context, uint8_t* out, size_t out_len),
+                                                 int(* add_entropy_cb)(void* context, const uint8_t input[], size_t length),
+                                                 void(* destroy_cb)(void* context));
+
 /**
 * Get random bytes from a random number generator
 * @param rng rng object

third_party/botan/src/lib/ffi/ffi_rng.cpp

@@ -1,5 +1,6 @@
 /*
 * (C) 2015,2017 Jack Lloyd
+* (C) 2021 René Fischer
 *
 * Botan is released under the Simplified BSD License (see license.txt)
 */
@@ -10,6 +11,8 @@
 #include <botan/system_rng.h>
 #include <botan/auto_rng.h>
 
+#include <functional>
+
 #if defined(BOTAN_HAS_PROCESSOR_RNG)
    #include <botan/processor_rng.h>
 #endif
@@ -57,6 +60,101 @@ int botan_rng_init(botan_rng_t* rng_out, const char* rng_type)
       });
    }
 
+int botan_rng_init_custom(botan_rng_t* rng_out, const char* rng_name, void* context,
+                          int(* get_cb)(void* context, uint8_t* out, size_t out_len),
+                          int(* add_entropy_cb)(void* context, const uint8_t input[], size_t length),
+                          void(* destroy_cb)(void* context))
+{
+return ffi_guard_thunk(__func__,[=]() -> int {
+   if(rng_out == nullptr)
+      return BOTAN_FFI_ERROR_NULL_POINTER;
+
+   if(rng_name == nullptr)
+      return BOTAN_FFI_ERROR_NULL_POINTER;
+
+   if(get_cb == nullptr)
+      return BOTAN_FFI_ERROR_NULL_POINTER;
+
+   class Custom_RNG : public Botan::RandomNumberGenerator
+      {
+      public:
+         Custom_RNG(const std::string& name, void* context,
+                    int(* get_cb)(void* context, uint8_t* out, size_t out_len),
+                    int(* add_entropy_cb)(void* context, const uint8_t input[], size_t length),
+                    void(* destroy_cb)(void* context)) :
+            m_name(name)
+            {
+               m_context = context;
+               m_get_cb = get_cb;
+               m_add_entropy_cb = add_entropy_cb;
+               m_destroy_cb = destroy_cb;
+            }
+
+         ~Custom_RNG()
+         {
+            if(m_destroy_cb)
+            {
+               m_destroy_cb(m_context);
+            }
+         }
+
+         void randomize(uint8_t output[], size_t length) override
+         {
+            int rc = m_get_cb(m_context, output, length);
+            if(rc)
+            {
+               throw Botan::Invalid_State("Failed to get random from C callback, rc=" + std::to_string(rc));
+            }
+         }
+
+         bool accepts_input() const override
+         {
+            return m_add_entropy_cb != nullptr;
+         }
+
+         void add_entropy(const uint8_t input[], size_t length) override
+         {
+            if(m_add_entropy_cb == nullptr)
+            {
+               return;
+            }
+
+            int rc = m_add_entropy_cb(m_context, input, length);
+            if(rc)
+            {
+               throw Botan::Invalid_State("Failed to add entropy via C callback, rc=" + std::to_string(rc));
+            }
+         }
+
+         std::string name() const override
+         {
+            return m_name;
+         }
+
+         void clear() override
+         {
+         }
+
+         bool is_seeded() const override
+         {
+            return true;
+         }
+
+      private:
+         std::string m_name;
+         void* m_context;
+         std::function<int(void* context, uint8_t* out, size_t out_len)> m_get_cb;
+         std::function<int(void* context, const uint8_t input[], size_t length)> m_add_entropy_cb;
+         std::function<void(void* context)> m_destroy_cb;
+   };
+
+   std::unique_ptr<Botan::RandomNumberGenerator> rng(new Custom_RNG(rng_name, context, get_cb, add_entropy_cb, destroy_cb));
+
+   *rng_out = new botan_rng_struct(rng.release());
+   return BOTAN_FFI_SUCCESS;
+   });
+}
+
 int botan_rng_destroy(botan_rng_t rng)
    {
    return BOTAN_FFI_CHECKED_DELETE(rng);

third_party/botan/src/lib/ffi/info.txt

@@ -1,5 +1,5 @@
 <defines>
-FFI -> 20191214
+FFI -> 20210220
 </defines>
 
 <header:internal>

third_party/botan/src/lib/hash/sha3/sha3.cpp

@@ -11,23 +11,47 @@
 #include <botan/exceptn.h>
 #include <botan/cpuid.h>
 
+#include <tuple>
+
 namespace Botan {
 
 namespace {
 
+// This is a workaround for a suspected bug in clang 12 (and XCode 13)
+// that caused a miscompile of the SHA3 implementation for optimization
+// level -O2 and higher.
+//
+// For details, see: https://github.com/randombit/botan/issues/2802
+#if    defined(__clang__) && \
+    (( defined(__apple_build_version__) && __clang_major__ == 13) || \
+     (!defined(__apple_build_version__) && __clang_major__ == 12))
+#define BOTAN_WORKAROUND_MAYBE_INLINE __attribute__((noinline))
+#else
+#define BOTAN_WORKAROUND_MAYBE_INLINE inline
+#endif
+
+BOTAN_WORKAROUND_MAYBE_INLINE std::tuple<uint64_t, uint64_t, uint64_t, uint64_t, uint64_t>
+   xor_CNs(const uint64_t A[25])
+   {
+   return {
+      A[0] ^ A[5] ^ A[10] ^ A[15] ^ A[20],
+      A[1] ^ A[6] ^ A[11] ^ A[16] ^ A[21],
+      A[2] ^ A[7] ^ A[12] ^ A[17] ^ A[22],
+      A[3] ^ A[8] ^ A[13] ^ A[18] ^ A[23],
+      A[4] ^ A[9] ^ A[14] ^ A[19] ^ A[24]};
+   }
+
+#undef BOTAN_WORKAROUND_MAYBE_INLINE
+
 inline void SHA3_round(uint64_t T[25], const uint64_t A[25], uint64_t RC)
    {
-   const uint64_t C0 = A[0] ^ A[5] ^ A[10] ^ A[15] ^ A[20];
+   const auto Cs = xor_CNs(A);
-   const uint64_t C1 = A[1] ^ A[6] ^ A[11] ^ A[16] ^ A[21];
-   const uint64_t C2 = A[2] ^ A[7] ^ A[12] ^ A[17] ^ A[22];
-   const uint64_t C3 = A[3] ^ A[8] ^ A[13] ^ A[18] ^ A[23];
-   const uint64_t C4 = A[4] ^ A[9] ^ A[14] ^ A[19] ^ A[24];
 
-   const uint64_t D0 = rotl<1>(C0) ^ C3;
+   const uint64_t D0 = rotl<1>(std::get<0>(Cs)) ^ std::get<3>(Cs);
-   const uint64_t D1 = rotl<1>(C1) ^ C4;
+   const uint64_t D1 = rotl<1>(std::get<1>(Cs)) ^ std::get<4>(Cs);
-   const uint64_t D2 = rotl<1>(C2) ^ C0;
+   const uint64_t D2 = rotl<1>(std::get<2>(Cs)) ^ std::get<0>(Cs);
-   const uint64_t D3 = rotl<1>(C3) ^ C1;
+   const uint64_t D3 = rotl<1>(std::get<3>(Cs)) ^ std::get<1>(Cs);
-   const uint64_t D4 = rotl<1>(C4) ^ C2;
+   const uint64_t D4 = rotl<1>(std::get<4>(Cs)) ^ std::get<2>(Cs);
 
    const uint64_t B00 =          A[ 0] ^ D1;
    const uint64_t B01 = rotl<44>(A[ 6] ^ D2);

third_party/botan/src/lib/hash/sha3/sha3_bmi2/sha3_bmi2.cpp

@@ -8,23 +8,47 @@
 #include <botan/sha3.h>
 #include <botan/rotate.h>
 
+#include <tuple>
+
 namespace Botan {
 
 namespace {
 
+// This is a workaround for a suspected bug in clang 12 (and XCode 13)
+// that caused a miscompile of the SHA3 implementation for optimization
+// level -O2 and higher.
+//
+// For details, see: https://github.com/randombit/botan/issues/2802
+#if    defined(__clang__) && \
+    (( defined(__apple_build_version__) && __clang_major__ == 13) || \
+     (!defined(__apple_build_version__) && __clang_major__ == 12))
+#define BOTAN_WORKAROUND_MAYBE_INLINE __attribute__((noinline))
+#else
+#define BOTAN_WORKAROUND_MAYBE_INLINE inline
+#endif
+
+BOTAN_WORKAROUND_MAYBE_INLINE std::tuple<uint64_t, uint64_t, uint64_t, uint64_t, uint64_t>
+   xor_CNs(const uint64_t A[25])
+   {
+   return {
+      A[0] ^ A[5] ^ A[10] ^ A[15] ^ A[20],
+      A[1] ^ A[6] ^ A[11] ^ A[16] ^ A[21],
+      A[2] ^ A[7] ^ A[12] ^ A[17] ^ A[22],
+      A[3] ^ A[8] ^ A[13] ^ A[18] ^ A[23],
+      A[4] ^ A[9] ^ A[14] ^ A[19] ^ A[24]};
+   }
+
+#undef BOTAN_WORKAROUND_MAYBE_INLINE
+
 inline void SHA3_BMI2_round(uint64_t T[25], const uint64_t A[25], uint64_t RC)
    {
-   const uint64_t C0 = A[0] ^ A[5] ^ A[10] ^ A[15] ^ A[20];
+   const auto Cs = xor_CNs(A);
-   const uint64_t C1 = A[1] ^ A[6] ^ A[11] ^ A[16] ^ A[21];
-   const uint64_t C2 = A[2] ^ A[7] ^ A[12] ^ A[17] ^ A[22];
-   const uint64_t C3 = A[3] ^ A[8] ^ A[13] ^ A[18] ^ A[23];
-   const uint64_t C4 = A[4] ^ A[9] ^ A[14] ^ A[19] ^ A[24];
 
-   const uint64_t D0 = rotl<1>(C0) ^ C3;
+   const uint64_t D0 = rotl<1>(std::get<0>(Cs)) ^ std::get<3>(Cs);
-   const uint64_t D1 = rotl<1>(C1) ^ C4;
+   const uint64_t D1 = rotl<1>(std::get<1>(Cs)) ^ std::get<4>(Cs);
-   const uint64_t D2 = rotl<1>(C2) ^ C0;
+   const uint64_t D2 = rotl<1>(std::get<2>(Cs)) ^ std::get<0>(Cs);
-   const uint64_t D3 = rotl<1>(C3) ^ C1;
+   const uint64_t D3 = rotl<1>(std::get<3>(Cs)) ^ std::get<1>(Cs);
-   const uint64_t D4 = rotl<1>(C4) ^ C2;
+   const uint64_t D4 = rotl<1>(std::get<4>(Cs)) ^ std::get<2>(Cs);
 
    const uint64_t B00 =          A[ 0] ^ D1;
    const uint64_t B01 = rotl<44>(A[ 6] ^ D2);

third_party/botan/src/lib/math/bigint/bigint.cpp

@@ -165,8 +165,8 @@ bool BigInt::is_less_than(const BigInt& other) const
 
    if(other.is_negative() && this->is_negative())
       {
-      return !bigint_ct_is_lt(other.data(), other.sig_words(),
+      return bigint_ct_is_lt(other.data(), other.sig_words(),
-                              this->data(), this->sig_words(), true).is_set();
+                             this->data(), this->sig_words()).is_set();
       }
 
    return bigint_ct_is_lt(this->data(), this->sig_words(),

third_party/botan/src/lib/math/numbertheory/numthry.cpp

@@ -61,13 +61,29 @@ size_t low_zero_bits(const BigInt& n)
    return seen_nonempty_word.if_set_return(low_zero);
    }
 
+namespace {
+
+size_t safegcd_loop_bound(size_t f_bits, size_t g_bits)
+   {
+   const size_t d = std::max(f_bits, g_bits);
+
+   if(d < 46)
+      return (49*d + 80) / 17;
+   else
+      return (49*d + 57) / 17;
+   }
+
+}
+
 /*
 * Calculate the GCD
 */
 BigInt gcd(const BigInt& a, const BigInt& b)
    {
-   if(a.is_zero() || b.is_zero())
+   if(a.is_zero())
-      return 0;
+      return abs(b);
+   if(b.is_zero())
+      return abs(a);
    if(a == 1 || b == 1)
       return 1;
 
@@ -91,7 +107,7 @@ BigInt gcd(const BigInt& a, const BigInt& b)
 
    int32_t delta = 1;
 
-   const size_t loop_cnt = 4 + 3*std::max(f.bits(), g.bits());
+   const size_t loop_cnt = safegcd_loop_bound(f.bits(), g.bits());
 
    BigInt newg, t;
    for(size_t i = 0; i != loop_cnt; ++i)
@@ -100,8 +116,8 @@ BigInt gcd(const BigInt& a, const BigInt& b)
 
       const bool need_swap = (g.is_odd() && delta > 0);
 
-      // if(need_swap) delta *= -1
+      // if(need_swap) { delta *= -1 } else { delta *= 1 }
-      delta *= CT::Mask<uint8_t>::expand(need_swap).select(0, 2) - 1;
+      delta *= CT::Mask<uint8_t>::expand(need_swap).if_not_set_return(2) - 1;
       f.ct_cond_swap(need_swap, g);
       g.ct_cond_swap(need_swap, newg);
 
@@ -116,6 +132,8 @@ BigInt gcd(const BigInt& a, const BigInt& b)
    f.const_time_unpoison();
    g.const_time_unpoison();
 
+   BOTAN_ASSERT_NOMSG(g.is_zero());
+
    return f;
    }
 

third_party/botan/src/lib/pk_pad/eme_oaep/oaep.cpp

@@ -72,9 +72,9 @@ secure_vector<uint8_t> OAEP::unpad(uint8_t& valid_mask,
    Therefore, the first byte can always be skipped safely.
    */
 
-   const uint8_t skip_first = CT::Mask<uint8_t>::is_zero(in[0]).if_set_return(1);
+   const auto leading_0 = CT::Mask<uint8_t>::is_zero(in[0]);
 
-   secure_vector<uint8_t> input(in + skip_first, in + in_length);
+   secure_vector<uint8_t> input(in + 1, in + in_length);
 
    const size_t hlen = m_Phash.size();
 
@@ -86,7 +86,9 @@ secure_vector<uint8_t> OAEP::unpad(uint8_t& valid_mask,
              input.data(), hlen,
              &input[hlen], input.size() - hlen);
 
-   return oaep_find_delim(valid_mask, input.data(), input.size(), m_Phash);
+   auto unpadded = oaep_find_delim(valid_mask, input.data(), input.size(), m_Phash);
+   valid_mask &= leading_0.unpoisoned_value();
+   return unpadded;
    }
 
 secure_vector<uint8_t>

third_party/botan/src/lib/prov/openssl/openssl_block.cpp

@@ -223,27 +223,10 @@ make_openssl_block_cipher(const std::string& name)
 #endif
 
 #if defined(BOTAN_HAS_DES) && !defined(OPENSSL_NO_DES)
-   if(name == "DES")
-      return MAKE_OPENSSL_BLOCK(EVP_des_ecb);
    if(name == "TripleDES")
       return MAKE_OPENSSL_BLOCK_KEYLEN(EVP_des_ede3_ecb, 16, 24, 8);
 #endif
 
-#if defined(BOTAN_HAS_BLOWFISH) && !defined(OPENSSL_NO_BF)
-   if(name == "Blowfish")
-      return MAKE_OPENSSL_BLOCK_KEYLEN(EVP_bf_ecb, 1, 56, 1);
-#endif
-
-#if defined(BOTAN_HAS_CAST_128) && !defined(OPENSSL_NO_CAST)
-   if(name == "CAST-128")
-      return MAKE_OPENSSL_BLOCK_KEYLEN(EVP_cast5_ecb, 1, 16, 1);
-#endif
-
-#if defined(BOTAN_HAS_SEED) && !defined(OPENSSL_NO_SEED)
-   if(name == "SEED")
-      return MAKE_OPENSSL_BLOCK(EVP_seed_ecb);
-#endif
-
    return nullptr;
    }
 

third_party/botan/src/lib/prov/openssl/openssl_hash.cpp

@@ -125,26 +125,11 @@ make_openssl_hash(const std::string& name)
       return MAKE_OPENSSL_HASH(EVP_sha1);
 #endif
 
-#if defined(BOTAN_HAS_RIPEMD_160) && !defined(OPENSSL_NO_RIPEMD)
-   if(name == "RIPEMD-160")
-      return MAKE_OPENSSL_HASH(EVP_ripemd160);
-#endif
-
 #if defined(BOTAN_HAS_MD5) && !defined(OPENSSL_NO_MD5)
    if(name == "MD5")
       return MAKE_OPENSSL_HASH(EVP_md5);
    #endif
 
-#if defined(BOTAN_HAS_MD4) && !defined(OPENSSL_NO_MD4)
-   if(name == "MD4")
-      return MAKE_OPENSSL_HASH(EVP_md4);
-#endif
-
-#if defined(BOTAN_HAS_WHIRLPOOL) && !defined(OPENSSL_NO_WHIRLPOOL)
-   if(name == "Whirlpool")
-      return MAKE_OPENSSL_HASH(EVP_whirlpool);
-#endif
-
    return nullptr;
    }
 

third_party/botan/src/lib/pubkey/elgamal/elgamal.cpp

@@ -108,8 +108,14 @@ ElGamal_Encryption_Operation::raw_encrypt(const uint8_t msg[], size_t msg_len,
    if(m >= m_group.get_p())
       throw Invalid_Argument("ElGamal encryption: Input is too large");
 
-   const size_t k_bits = m_group.exponent_bits();
+   /*
-   const BigInt k(rng, k_bits);
+   Some ElGamal implementations generate keys where using short exponents
+   is unsafe. Always use full length exponents to avoid this.
+
+   See https://eprint.iacr.org/2021/923 for details.
+   */
+   const size_t k_bits = m_group.p_bits() - 1;
+   const BigInt k(rng, k_bits, false);
 
    const BigInt a = m_group.power_g_p(k, k_bits);
    const BigInt b = m_group.multiply_mod_p(m, monty_execute(*m_monty_y_p, k, k_bits));

third_party/botan/src/lib/tls/asio/asio_stream.h

@@ -334,28 +334,36 @@ class Stream
 
    private:
       /**
-       * @brief Internal wrapper type to adapt the expected signature of `async_shutdown`
+       * @brief Internal wrapper type to adapt the expected signature of `async_shutdown` to the completion handler
-       *        to the completion handler signature of `AsyncWriteOperation`.
+       *        signature of `AsyncWriteOperation`.
        *
-       * This is boilerplate to ignore the `size_t` parameter that is passed to the
+       * This is boilerplate to ignore the `size_t` parameter that is passed to the completion handler of
-       * completion handler of `AsyncWriteOperation`.
+       * `AsyncWriteOperation`. Note that it needs to retain the wrapped handler's executor.
-       *
-       * @todo in C++14 and above this could be implemented as a mutable lambda expression
-       *       that captures `handler` by perfect forwarding, like so:
-       *
-       *       [h = std::forward<Handler>(handler)] (...) mutable { return h(ec); }
        */
-      template <typename Handler>
+      template <typename Handler, typename Executor>
-      class Wrapper
+      struct Wrapper
          {
-         public:
+         void operator()(boost::system::error_code ec, std::size_t)
-            Wrapper(Handler&& handler) : _handler(std::forward<Handler>(handler)) {}
-            void operator()(boost::system::error_code ec, size_t)
             {
-               _handler(ec);
+            handler(ec);
             }
-         private:
+
-            Handler _handler;
+         using executor_type = boost::asio::associated_executor_t<Handler, Executor>;
+
+         executor_type get_executor() const noexcept
+            {
+            return boost::asio::get_associated_executor(handler, io_executor);
+            }
+
+         using allocator_type = boost::asio::associated_allocator_t<Handler>;
+
+         allocator_type get_allocator() const noexcept
+            {
+            return boost::asio::get_associated_allocator(handler);
+            }
+
+         Handler handler;
+         Executor io_executor;
          };
 
    public:
@@ -380,9 +388,9 @@ class Stream
          // If ec is set by native_handle->close(), the AsyncWriteOperation created below will do nothing but call the
          // handler with the error_code set appropriately - no need to early return here.
 
-         using ShutdownHandlerWrapper = Wrapper<ShutdownHandler>;
+         using ShutdownHandlerWrapper = Wrapper<ShutdownHandler, typename Stream::executor_type>;
 
-         ShutdownHandlerWrapper w(std::forward<ShutdownHandler>(handler));
+         ShutdownHandlerWrapper w{std::forward<ShutdownHandler>(handler), get_executor()};
          BOOST_ASIO_SHUTDOWN_HANDLER_CHECK(ShutdownHandler, w) type_check;
 
          boost::asio::async_completion<ShutdownHandlerWrapper, void(boost::system::error_code, std::size_t)>

third_party/botan/src/lib/tls/tls_record.cpp

@@ -340,11 +340,13 @@ Record_Header read_tls_record(secure_vector<uint8_t>& readbuf,
       BOTAN_ASSERT_EQUAL(readbuf.size(), TLS_HEADER_SIZE, "Have an entire header");
       }
 
-   const Protocol_Version version(readbuf[1], readbuf[2]);
+   if(readbuf[1] != 3)
-
+      {
-   if(version.is_datagram_protocol())
       throw TLS_Exception(Alert::PROTOCOL_VERSION,
-                          "Expected TLS but got a record with DTLS version");
+                          "Got unexpected TLS record version");
+      }
+
+   const Protocol_Version version(readbuf[1], readbuf[2]);
 
    const size_t record_size = make_uint16(readbuf[TLS_HEADER_SIZE-2],
                                           readbuf[TLS_HEADER_SIZE-1]);

third_party/botan/src/lib/utils/parsing.cpp

@@ -324,8 +324,6 @@ std::string replace_char(const std::string& str, char from_char, char to_char)
    return out;
    }
 
-namespace {
-
 std::string tolower_string(const std::string& in)
    {
    std::string s = in;
@@ -338,8 +336,6 @@ std::string tolower_string(const std::string& in)
    return s;
    }
 
-}
-
 bool host_wildcard_match(const std::string& issued_, const std::string& host_)
    {
    const std::string issued = tolower_string(issued_);

third_party/botan/src/lib/utils/parsing.h

@@ -167,6 +167,8 @@ std::map<std::string, std::string> BOTAN_PUBLIC_API(2,8) read_kv(const std::stri
 
 std::string BOTAN_PUBLIC_API(2,0) clean_ws(const std::string& s);
 
+std::string tolower_string(const std::string& s);
+
 /**
 * Check if the given hostname is a match for the specified wildcard
 */

third_party/botan/src/lib/utils/rotate.h

@@ -69,14 +69,20 @@ inline T rotr_var(T input, size_t rot)
 template<>
 inline uint32_t rotl_var(uint32_t input, size_t rot)
    {
-   asm("roll %1,%0" : "+r" (input) : "c" (static_cast<uint8_t>(rot)));
+   asm("roll %1,%0"
+       : "+r" (input)
+       : "c" (static_cast<uint8_t>(rot))
+       : "cc");
    return input;
    }
 
 template<>
 inline uint32_t rotr_var(uint32_t input, size_t rot)
    {
-   asm("rorl %1,%0" : "+r" (input) : "c" (static_cast<uint8_t>(rot)));
+   asm("rorl %1,%0"
+       : "+r" (input)
+       : "c" (static_cast<uint8_t>(rot))
+       : "cc");
    return input;
    }
 

third_party/botan/src/lib/x509/name_constraint.cpp

@@ -165,17 +165,19 @@ bool GeneralName::matches_dns(const std::string& nam) const
    {
    if(nam.size() == name().size())
       {
-      return nam == name();
+      return tolower_string(nam) == tolower_string(name());
       }
    else if(name().size() > nam.size())
       {
+      // The constraint is longer than the issued name: not possibly a match
       return false;
       }
    else // name.size() < nam.size()
       {
-      std::string constr = name().front() == '.' ? name() : "." + name();
       // constr is suffix of nam
-      return constr == nam.substr(nam.size() - constr.size(), constr.size());
+      const std::string constr = name().front() == '.' ? name() : "." + name();
+      const std::string substr = nam.substr(nam.size() - constr.size(), constr.size());
+      return tolower_string(constr) == tolower_string(substr);
       }
    }
 

third_party/botan/src/lib/x509/x509_dn.cpp

@@ -291,7 +291,7 @@ void X509_DN::decode_from(BER_Decoder& source)
          rdn.start_cons(SEQUENCE)
             .decode(oid)
             .decode(str) // TODO support Any
-            .end_cons().verify_end("Invalid X509_DN, data follows RDN");
+            .end_cons();
 
          add_attribute(oid, str);
          }

third_party/botan/src/lib/x509/x509_ext.cpp

@@ -657,20 +657,17 @@ void Name_Constraints::validate(const X509_Certificate& subject, const X509_Cert
    {
    if(!m_name_constraints.permitted().empty() || !m_name_constraints.excluded().empty())
       {
-      if(!subject.is_CA_cert() || !subject.is_critical("X509v3.NameConstraints"))
+      if(!subject.is_CA_cert())
+         {
          cert_status.at(pos).insert(Certificate_Status_Code::NAME_CONSTRAINT_ERROR);
+         }
 
       const bool issuer_name_constraint_critical =
          issuer.is_critical("X509v3.NameConstraints");
 
-      const bool at_self_signed_root = (pos == cert_path.size() - 1);
-
       // Check that all subordinate certs pass the name constraint
-      for(size_t j = 0; j <= pos; ++j)
+      for(size_t j = 0; j < pos; ++j)
          {
-         if(pos == j && at_self_signed_root)
-            continue;
-
          bool permitted = m_name_constraints.permitted().empty();
          bool failed = false;
 

third_party/botan/src/scripts/ci/appveyor.yml

@@ -19,6 +19,7 @@ environment:
       MAKE_TOOL: nmake
       TARGET_CC: msvc
       EXTRA_FLAGS: "--disable-werror"
+      DISABLED_TESTS: "certstor_system" # requires 'ISRG Root X1' / not in this AppVeyor image
 
     # MSVC 2017 DLL x86-32
     - CC: VC2017
@@ -30,6 +31,7 @@ environment:
       BOOST_SYSTEM_LIBRARY: "libboost_system-vc141-mt-x32-1_69"
       MAKE_TOOL: jom
       TARGET_CC: msvc
+      DISABLED_TESTS: "certstor_system" # requires 'ISRG Root X1' / not in this AppVeyor image
 
     # MSVC 2017 DLL x86-64
     - CC: VC2017
@@ -41,6 +43,7 @@ environment:
       BOOST_SYSTEM_LIBRARY: "libboost_system-vc141-mt-x64-1_69"
       MAKE_TOOL: jom
       TARGET_CC: msvc
+      DISABLED_TESTS: "certstor_system" # requires 'ISRG Root X1' / not in this AppVeyor image
 
     # MSVC 2017 static x86-64
     - CC: VC2017
@@ -53,6 +56,7 @@ environment:
       MAKE_TOOL: jom
       TARGET_CC: msvc
       EXTRA_FLAGS: "--extra-cxxflags=/DUNICODE --extra-cxxflags=/D_UNICODE"
+      DISABLED_TESTS: "certstor_system" # requires 'ISRG Root X1' / not in this AppVeyor image
 
     # MSVC 2019 static x86-64 w/debug iterators
     - CC: VC2019
@@ -62,19 +66,11 @@ environment:
       MAKE_TOOL: jom
       TARGET_CC: msvc
 
-    # MSVC 2019 x86-64 preview
-    - CC: VC2019p
-      PLATFORM: x86_amd64
-      TARGET: static
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 Preview
-      MAKE_TOOL: jom
-      TARGET_CC: msvc
-
     # MinGW GCC
     - CC: MinGW
       PLATFORM: x86_amd64
       TARGET: static
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
       MAKE_TOOL: mingw32-make
       TARGET_CC: gcc
 
@@ -82,7 +78,7 @@ install:
   - call src\scripts\ci\setup_appveyor.bat
 
 build_script:
-  - python src\scripts\ci_build.py --os=windows --cc=%TARGET_CC% --without-python3 --compiler-cache=sccache --make-tool=%MAKE_TOOL% --cpu=%PLATFORM% %EXTRA_FLAGS% %TARGET%
+  - python src\scripts\ci_build.py --os=windows --cc=%TARGET_CC% --without-python3 --compiler-cache=sccache --make-tool=%MAKE_TOOL% --cpu=%PLATFORM% --disabled-tests=%DISABLED_TESTS% %EXTRA_FLAGS% %TARGET%
 
 # whitelist branches to avoid testing feature branches twice (as branch and as pull request)
 branches:

third_party/botan/src/scripts/ci/setup_appveyor.bat

@@ -4,7 +4,6 @@ echo Current build setup CC="%CC%" PLATFORM="%PLATFORM%" TARGET="%TARGET%"
 if %CC% == VC2015 call "%ProgramFiles(x86)%\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %PLATFORM%
 if %CC% == VC2017 call "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" %PLATFORM%
 if %CC% == VC2019 call "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" %PLATFORM%
-if %CC% == VC2019p call "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Preview\VC\Auxiliary\Build\vcvarsall.bat" %PLATFORM%
 if %CC% == MinGW set PATH=%PATH%;C:\msys64\mingw64\bin
 
 rem check compiler version

third_party/botan/src/scripts/ci/setup_gh_actions.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# GitHub Actions setup script for Botan build
+#
+# (C) 2015,2017 Simon Warta
+# (C) 2016,2017,2018,2020 Jack Lloyd
+#
+# Botan is released under the Simplified BSD License (see license.txt)
+
+command -v shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
+
+set -ex
+
+TARGET=$1
+
+if type -p "apt-get"; then
+    sudo apt-get -qq update
+    sudo apt-get -qq install ccache
+
+    if [ "$TARGET" = "valgrind" ]; then
+        sudo apt-get -qq install valgrind
+
+    elif [ "$TARGET" = "clang" ]; then
+        sudo apt-get -qq install clang
+
+    elif [ "$TARGET" = "cross-i386" ]; then
+        sudo apt-get -qq install g++-multilib linux-libc-dev libc6-dev-i386
+
+    elif [ "$TARGET" = "cross-win64" ]; then
+        sudo apt-get -qq install wine-development g++-mingw-w64-x86-64
+
+    elif [ "$TARGET" = "cross-arm64" ]; then
+        sudo apt-get -qq install qemu-user g++-aarch64-linux-gnu
+
+    elif [ "$TARGET" = "cross-ppc64" ]; then
+        sudo apt-get -qq install qemu-user g++-powerpc64le-linux-gnu
+
+    elif [ "$TARGET" = "cross-android-arm32" ] || [ "$TARGET" = "cross-android-arm64" ]; then
+        wget -nv https://dl.google.com/android/repository/"$ANDROID_NDK"-linux-x86_64.zip
+        unzip -qq "$ANDROID_NDK"-linux-x86_64.zip
+
+    elif [ "$TARGET" = "baremetal" ]; then
+        sudo apt-get -qq install gcc-arm-none-eabi libstdc++-arm-none-eabi-newlib
+
+        echo 'extern "C" void __sync_synchronize() {}' >> src/tests/main.cpp
+        echo 'extern "C" void __sync_synchronize() {}' >> src/cli/main.cpp
+
+    elif [ "$TARGET" = "lint" ]; then
+        sudo apt-get -qq install pylint
+
+    elif [ "$TARGET" = "coverage" ]; then
+        sudo apt-get -qq install g++-8 softhsm2 libtspi-dev lcov python-coverage libboost-all-dev gdb
+        pip install --user codecov
+        echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+        git clone --depth 1 --branch runner-changes https://github.com/randombit/boringssl.git
+
+        sudo chgrp -R "$(id -g)" /var/lib/softhsm/ /etc/softhsm
+        sudo chmod g+w /var/lib/softhsm/tokens
+
+        softhsm2-util --init-token --free --label test --pin 123456 --so-pin 12345678
+        echo "PKCS11_LIB=/usr/lib/softhsm/libsofthsm2.so" >> "$GITHUB_ENV"
+
+    elif [ "$TARGET" = "docs" ]; then
+        sudo apt-get -qq install doxygen python-docutils python3-sphinx
+    fi
+else
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache
+fi

third_party/botan/src/scripts/ci_build.py

@@ -2,7 +2,8 @@
 
 """
 CI build script
-(C) 2017 Jack Lloyd
+(C) 2017,2020 Jack Lloyd
+
 Botan is released under the Simplified BSD License (see license.txt)
 """
 
@@ -24,9 +25,9 @@ def get_concurrency():
         return def_concurrency
 
 def build_targets(target, target_os):
-    if target in ['shared', 'mini-shared', 'bsi', 'nist']:
+    if target in ['shared', 'minimized', 'bsi', 'nist']:
         yield 'shared'
-    elif target in ['static', 'mini-static', 'fuzzers', 'baremetal']:
+    elif target in ['static', 'fuzzers', 'baremetal']:
         yield 'static'
     elif target_os in ['windows']:
         yield 'shared'
@@ -43,7 +44,8 @@ def build_targets(target, target_os):
         yield 'bogo_shim'
 
 def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
-                    ccache, root_dir, pkcs11_lib, use_gdb, disable_werror, extra_cxxflags):
+                    ccache, root_dir, pkcs11_lib, use_gdb, disable_werror, extra_cxxflags,
+                    disabled_tests):
     # pylint: disable=too-many-branches,too-many-statements,too-many-arguments,too-many-locals
 
     """
@@ -51,7 +53,7 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
     """
     is_cross_target = target.startswith('cross-')
 
-    if target_os not in ['linux', 'osx', 'windows']:
+    if target_os not in ['linux', 'osx', 'windows', 'freebsd']:
         print('Error unknown OS %s' % (target_os))
         return (None, None, None)
 
@@ -73,20 +75,17 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
     test_prefix = []
     test_cmd = [os.path.join(root_dir, 'botan-test')]
 
-    essential_tests = ['block', 'aead', 'hash', 'stream', 'mac', 'modes', 'kdf',
+    install_prefix = tempfile.mkdtemp(prefix='botan-install-')
-                       'hmac_drbg', 'hmac_drbg_unit', 'tls',
-                       'rsa_sign', 'rsa_verify', 'dh_kat',
-                       'ecc_randomized', 'ecdh_kat', 'ecdsa_sign', 'curve25519_scalar',
-                       'cpuid', 'simd_32', 'os_utils', 'util', 'util_dates']
-
-    install_prefix = os.path.join(tempfile.gettempdir(), 'botan-install')
 
     flags = ['--prefix=%s' % (install_prefix),
              '--cc=%s' % (target_cc),
              '--os=%s' % (target_os),
              '--build-targets=%s' % ','.join(build_targets(target, target_os))]
 
-    if not disable_werror:
+    if ccache is not None:
+        flags += ['--no-store-vc-rev', '--compiler-cache=%s' % (ccache)]
+
+    if target_os != 'osx' and not disable_werror:
         flags += ['--werror-mode']
 
     if target_cpu is not None:
@@ -95,18 +94,12 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
     for flag in extra_cxxflags:
         flags += ['--extra-cxxflags=%s' % (flag)]
 
-    if target in ['mini-static', 'mini-shared']:
+    if target in ['minimized']:
         flags += ['--minimized-build', '--enable-modules=system_rng,sha2_32,sha2_64,aes']
 
-    if target == 'static':
-        # Arbitrarily test amalgamation with the static lib builds
-        flags += ['--amalgamation']
-
     if target in ['bsi', 'nist']:
-        # Arbitrarily test disable static on module policy builds
         # tls is optional for bsi/nist but add it so verify tests work with these minimized configs
-        flags += ['--module-policy=%s' % (target),
+        flags += ['--module-policy=%s' % (target), '--enable-modules=tls']
-                  '--enable-modules=tls']
 
     if target == 'docs':
         flags += ['--with-doxygen', '--with-sphinx', '--with-rst2man']
@@ -114,27 +107,31 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
 
     if target == 'cross-win64':
         # this test compiles under MinGW but fails when run under Wine
-        test_cmd += ['--skip-tests=certstor_system']
+        disabled_tests.append('certstor_system')
 
     if target == 'coverage':
         flags += ['--with-coverage-info', '--with-debug-info', '--test-mode']
-        test_cmd += ['--skip-tests=tls_stream_integration']
 
     if target == 'valgrind':
-        # valgrind in 16.04 has a bug with rdrand handling
+        flags += ['--with-valgrind']
-        flags += ['--with-valgrind', '--disable-rdrand']
         test_prefix = ['valgrind', '--error-exitcode=9', '-v', '--leak-check=full', '--show-reachable=yes']
         # valgrind is single threaded anyway
         test_cmd += ['--test-threads=1']
         # valgrind is slow
-        test_cmd += essential_tests
+        slow_tests = [
+            'cryptobox', 'dh_invalid', 'dh_kat', 'dh_keygen',
+            'dl_group_gen', 'dlies', 'dsa_param', 'ecc_basemul',
+            'ecdsa_verify_wycheproof', 'mce_keygen', 'passhash9',
+            'rsa_encrypt', 'rsa_pss', 'rsa_pss_raw', 'scrypt',
+            'srp6_kat', 'x509_path_bsi', 'xmss_keygen', 'xmss_sign',
+            'pbkdf', 'argon2', 'bcrypt', 'bcrypt_pbkdf', 'compression',
+            'ed25519_sign', 'elgamal_keygen', 'x509_path_rsa_pss']
+
+        disabled_tests += slow_tests
 
     if target == 'fuzzers':
         flags += ['--unsafe-fuzzer-mode']
 
-    if target in ['fuzzers', 'coverage', 'valgrind']:
-        flags += ['--with-debug-info']
-
     if target in ['fuzzers', 'coverage']:
         flags += ['--build-fuzzers=test']
 
@@ -149,9 +146,6 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
     if target in ['valgrind', 'sanitizer', 'fuzzers']:
         flags += ['--disable-modules=locking_allocator']
 
-    if target == 'parallel':
-        flags += ['--with-openmp']
-
     if target == 'baremetal':
         cc_bin = 'arm-none-eabi-c++'
         flags += ['--cpu=arm32', '--disable-neon', '--without-stack-protector', '--ldflags=-specs=nosys.specs']
@@ -161,9 +155,7 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
         if target_os == 'ios':
             make_prefix = ['xcrun', '--sdk', 'iphoneos']
             test_cmd = None
-            if target == 'cross-arm32':
+            if target == 'cross-ios-arm64':
-                flags += ['--cpu=armv7', '--cc-abi-flags=-arch armv7 -arch armv7s -stdlib=libc++']
-            elif target == 'cross-arm64':
                 flags += ['--cpu=arm64', '--cc-abi-flags=-arch arm64 -stdlib=libc++']
             else:
                 raise Exception("Unknown cross target '%s' for iOS" % (target))
@@ -197,9 +189,6 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
 
         elif target == 'cross-i386':
             flags += ['--cpu=x86_32']
-        elif target == 'cross-arm32':
-            flags += ['--cpu=armv7']
-            cc_bin = 'arm-linux-gnueabihf-g++'
 
         elif target == 'cross-win64':
             # MinGW in 16.04 is lacking std::mutex for unknown reason
@@ -209,10 +198,12 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
             test_cmd = [os.path.join(root_dir, 'botan-test.exe')] + test_cmd[1:]
             test_prefix = ['wine']
         else:
-            # Build everything but restrict what is run
+            if target == 'cross-arm32':
-            test_cmd += essential_tests
+                flags += ['--cpu=armv7']
-
+                cc_bin = 'arm-linux-gnueabihf-g++'
-            if target == 'cross-arm64':
+                # Currently arm32 CI only runs on native AArch64
+                #test_prefix = ['qemu-arm', '-L', '/usr/arm-linux-gnueabihf/']
+            elif target == 'cross-arm64':
                 flags += ['--cpu=aarch64']
                 cc_bin = 'aarch64-linux-gnu-g++'
                 test_prefix = ['qemu-aarch64', '-L', '/usr/aarch64-linux-gnu/']
@@ -234,16 +225,13 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
     else:
         # Flags specific to native targets
 
-        if target == 'gcc4.8':
-            cc_bin = 'g++-4.8'
-
         if target_os in ['osx', 'linux']:
             flags += ['--with-bzip2', '--with-sqlite', '--with-zlib']
 
         if target_os in ['osx', 'ios']:
             flags += ['--with-commoncrypto']
 
-        if target_os == 'osx' or target == 'coverage':
+        if target == 'coverage':
             flags += ['--with-boost']
 
         if target_os == 'windows' and target in ['shared', 'static']:
@@ -262,32 +250,34 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
             flags += ['--with-lzma']
 
         if target_os == 'linux':
-            if target not in ['sanitizer', 'valgrind', 'mini-shared', 'mini-static']:
+            if target not in ['sanitizer', 'valgrind', 'minimized']:
                 # Avoid OpenSSL when using dynamic checkers, or on OS X where it sporadically
                 # is not installed on the CI image
                 flags += ['--with-openssl']
 
-        if target in ['sonar', 'coverage']:
+        if target in ['coverage']:
             flags += ['--with-tpm']
-            test_cmd += ['--run-long-tests', '--run-online-tests']
+            test_cmd += ['--run-online-tests']
             if pkcs11_lib and os.access(pkcs11_lib, os.R_OK):
                 test_cmd += ['--pkcs11-lib=%s' % (pkcs11_lib)]
 
-    if ccache is None:
+    if target in ['coverage', 'sanitizer']:
+        test_cmd += ['--run-long-tests']
+
     flags += ['--cc-bin=%s' % (cc_bin)]
-    elif ccache == 'clcache':
-        flags += ['--cc-bin=%s' % (ccache)]
-    else:
-        flags += ['--cc-bin=%s %s' % (ccache, cc_bin)]
-        # Avoid putting the revision in build.h, which helps ccache hit rates
-        flags += ['--no-store-vc-rev']
 
     if test_cmd is None:
         run_test_command = None
     else:
+        if use_gdb:
+            disabled_tests.append("os_utils")
+
+        # render 'disabled_tests' array into test_cmd
+        if disabled_tests:
+            test_cmd += ['--skip-tests=%s' % (','.join(disabled_tests))]
+
         if use_gdb:
             (cmd, args) = test_cmd[0], test_cmd[1:]
-            args += ['--skip-tests=os_utils']
             run_test_command = test_prefix + ['gdb', cmd,
                                               '-ex', 'run %s' % (' '.join(args)),
                                               '-ex', 'bt',
@@ -295,7 +285,7 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
         else:
             run_test_command = test_prefix + test_cmd
 
-    return flags, run_test_command, make_prefix, install_prefix
+    return flags, run_test_command, make_prefix
 
 def run_cmd(cmd, root_dir):
     """
@@ -339,13 +329,19 @@ def run_cmd(cmd, root_dir):
         if cmd[0] not in ['lcov']:
             sys.exit(proc.returncode)
 
+def default_os():
+    platform_os = platform.system().lower()
+    if platform_os == 'darwin':
+        return 'osx'
+    return platform_os
+
 def parse_args(args):
     """
     Parse arguments
     """
     parser = optparse.OptionParser()
 
-    parser.add_option('--os', default=platform.system().lower(),
+    parser.add_option('--os', default=default_os(),
                       help='Set the target os (default %default)')
     parser.add_option('--cc', default='gcc',
                       help='Set the target compiler type (default %default)')
@@ -369,22 +365,21 @@ def parse_args(args):
                       help='Build via amalgamation')
     parser.add_option('--disable-shared', action='store_true', default=False,
                       help='Disable building shared libraries')
+    parser.add_option('--disabled-tests', metavar='DISABLED_TESTS', default=[], action='append',
+                      help='Comma separated list of tests that should not be run')
 
     parser.add_option('--branch', metavar='B', default=None,
                       help='Specify branch being built')
 
-    parser.add_option('--add-travis-folds', action='store_true', default=False,
-                      help='Add fold markers for Travis UI')
-
     parser.add_option('--dry-run', action='store_true', default=False,
                       help='Just show commands to be executed')
     parser.add_option('--build-jobs', metavar='J', default=get_concurrency(),
                       help='Set number of jobs to run in parallel (default %default)')
 
     parser.add_option('--compiler-cache', default=None, metavar='CC',
-                      help='Set a compiler cache to use (ccache, sccache, clcache)')
+                      help='Set a compiler cache to use (ccache, sccache)')
 
-    parser.add_option('--pkcs11-lib', default=None, metavar='LIB',
+    parser.add_option('--pkcs11-lib', default=os.getenv('PKCS11_LIB'), metavar='LIB',
                       help='Set PKCS11 lib to use for testing')
 
     parser.add_option('--with-python3', dest='use_python3', action='store_true', default=None,
@@ -436,8 +431,6 @@ def main(args=None):
 
     target = args[1]
 
-    use_python2 = have_prog('python2')
-
     if options.use_python3 is None:
         use_python3 = have_prog('python3')
     else:
@@ -459,18 +452,12 @@ def main(args=None):
             return 1
 
     if options.compiler_cache is None and options.cc != 'msvc':
-        # Autodetect ccache, unless using clang profiling - ccache seems to misbehave there
+        # Autodetect ccache
-        if have_prog('ccache') and target not in ['sonar']:
+        if have_prog('ccache'):
             options.compiler_cache = 'ccache'
 
-    if options.compiler_cache == 'clcache' and target in ['sanitizer']:
+    if options.compiler_cache not in [None, 'ccache', 'sccache']:
-        # clcache doesn't support /Zi so using it just adds overhead with
+        raise Exception("Don't know about %s as a compiler cache" % (options.compiler_cache))
-        # no benefit
-        options.compiler_cache = None
-
-    if target == 'sonar' and os.getenv('SONAR_TOKEN') is None:
-        print('Skipping Sonar scan due to missing SONAR_TOKEN env variable')
-        return 0
 
     root_dir = options.root_dir
 
@@ -481,17 +468,9 @@ def main(args=None):
 
     if target == 'lint':
 
-        if not use_python2 and not use_python3:
-            raise Exception('No python interpreters found cannot lint')
-
         pylint_rc = '--rcfile=%s' % (os.path.join(root_dir, 'src/configs/pylint.rc'))
         pylint_flags = [pylint_rc, '--reports=no']
 
-        # Some disabled rules specific to Python2
-        # superfluous-parens: needed for Python3 compatible print statements
-        # too-many-locals: variable counting differs from pylint3
-        py2_flags = '--disable=superfluous-parens,too-many-locals'
-
         # Some disabled rules specific to Python3
         # useless-object-inheritance: complains about code still useful in Python2
         py3_flags = '--disable=useless-object-inheritance'
@@ -516,18 +495,15 @@ def main(args=None):
 
         full_paths = [os.path.join(root_dir, s) for s in py_scripts]
 
-        if use_python2:
-            cmds.append(['python2', '-m', 'pylint'] + pylint_flags + [py2_flags] + full_paths)
-
         if use_python3 and options.use_pylint3:
             cmds.append(['python3', '-m', 'pylint'] + pylint_flags + [py3_flags] + full_paths)
 
     else:
-        config_flags, run_test_command, make_prefix, install_prefix = determine_flags(
+        config_flags, run_test_command, make_prefix = determine_flags(
             target, options.os, options.cpu, options.cc,
             options.cc_bin, options.compiler_cache, root_dir,
             options.pkcs11_lib, options.use_gdb, options.disable_werror,
-            options.extra_cxxflags)
+            options.extra_cxxflags, options.disabled_tests)
 
         cmds.append([py_interp, os.path.join(root_dir, 'configure.py')] + config_flags)
 
@@ -541,28 +517,21 @@ def main(args=None):
         if target == 'docs':
             cmds.append(make_cmd + ['docs'])
         else:
-
+            if options.compiler_cache is not None:
-            ccache_show_stats = {
+                cmds.append([options.compiler_cache, '--show-stats'])
-                'ccache': '--show-stats',
-                'sccache': '--show-stats',
-                'clcache': '-s'
-            }
-
-            if options.compiler_cache in ccache_show_stats:
-                cmds.append([options.compiler_cache, ccache_show_stats[options.compiler_cache]])
 
             make_targets = ['libs', 'tests', 'cli']
 
             if target in ['coverage', 'fuzzers']:
-                make_targets += ['tests', 'cli', 'fuzzers', 'fuzzer_corpus_zip']
+                make_targets += ['fuzzer_corpus_zip', 'fuzzers']
 
             if target in ['coverage']:
                 make_targets += ['bogo_shim']
 
             cmds.append(make_prefix + make_cmd + make_targets)
 
-            if options.compiler_cache in ccache_show_stats:
+            if options.compiler_cache is not None:
-                cmds.append([options.compiler_cache, ccache_show_stats[options.compiler_cache]])
+                cmds.append([options.compiler_cache, '--show-stats'])
 
         if run_test_command is not None:
             cmds.append(run_test_command)
@@ -584,12 +553,11 @@ def main(args=None):
         if target in ['shared', 'coverage'] and options.os != 'windows':
             botan_exe = os.path.join(root_dir, 'botan-cli.exe' if options.os == 'windows' else 'botan')
 
+            args = ['--threads=%d' % (options.build_jobs)]
             test_scripts = ['test_cli.py', 'test_cli_crypt.py']
             for script in test_scripts:
-                cmds.append([py_interp,
+                cmds.append([py_interp, os.path.join(root_dir, 'src/scripts', script)] +
-                             os.path.join(root_dir, 'src/scripts', script),
+                            args + [botan_exe])
-                             '--threads=%d' % (options.build_jobs),
-                             botan_exe])
 
         python_tests = os.path.join(root_dir, 'src/scripts/test_python.py')
 
@@ -600,28 +568,15 @@ def main(args=None):
                     # Python on AppVeyor is a 32-bit binary so only test for 32-bit
                     cmds.append([py_interp, '-b', python_tests])
             else:
-                if use_python2:
-                    cmds.append(['python2', '-b', python_tests])
                 if use_python3:
                     cmds.append(['python3', '-b', python_tests])
 
         if target in ['shared', 'static', 'bsi', 'nist']:
             cmds.append(make_cmd + ['install'])
-            cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/ci_check_install.py'), install_prefix])
+            build_config = os.path.join(root_dir, 'build', 'build_config.json')
-
+            cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/ci_check_install.py'), build_config])
-        if target in ['sonar']:
-
-            cmds.append(['llvm-profdata', 'merge', '-sparse', 'default.profraw', '-o', 'botan.profdata'])
-            cmds.append(['llvm-cov', 'show', './botan-test',
-                         '-instr-profile=botan.profdata',
-                         '>', 'build/cov_report.txt'])
-            sonar_config = os.path.join(root_dir, 'src/configs/sonar-project.properties')
-            cmds.append(['sonar-scanner',
-                         '-Dproject.settings=%s' % (sonar_config),
-                         '-Dsonar.login=$SONAR_TOKEN'])
 
         if target in ['coverage']:
-
             if not have_prog('lcov'):
                 print('Error: lcov not found in PATH (%s)' % (os.getenv('PATH')))
                 return 1
@@ -644,7 +599,7 @@ def main(args=None):
                              python_tests])
 
             if have_prog('codecov'):
-                # If codecov exists assume we are on Travis and report to codecov.io
+                # If codecov exists assume we are in CI and report to codecov.io
                 cmds.append(['codecov', '>', 'codecov_stdout.log'])
             else:
                 # Otherwise generate a local HTML report

third_party/botan/src/scripts/ci_check_install.py

@@ -12,45 +12,93 @@ Botan is released under the Simplified BSD License (see license.txt)
 
 import os
 import sys
+import json
+import re
 
-def has_extension(filename, extensions):
+def verify_library(build_config):
-    for ext in [ext for ext in extensions]:
+    lib_dir = build_config['libdir']
-        if filename.endswith(".%s" % ext):
+    if not os.path.isdir(lib_dir):
-            return True
+        print('Error: libdir "%s" is not a directory' % lib_dir)
         return False
 
-def is_lib_file(filename):
+    found_libs = set([])
-    return has_extension(filename, ["so", "a", "dll", "dylib", "lib"])
 
-def is_header_file(filename):
+    major_version = int(build_config["version_major"])
-    return has_extension(filename, ["h", "hpp", "h++", "hxx", "hh"])
 
-def main():
+    if build_config['compiler'] == 'msvc':
-    if len(sys.argv) < 2:
+        expected_lib_format = r'^botan\.(dll|lib)$'
-        print("Usage: %s <install_prefix>" % sys.argv[0])
+    elif build_config['os'] == 'macos':
+        expected_lib_format = r'^libbotan-%d\.(a|dylib)$' % (major_version)
+    else:
+        expected_lib_format = r'^libbotan-%d\.(a|so)$' % (major_version)
+
+    lib_re = re.compile(expected_lib_format)
+
+    # Unlike the include dir this may have other random libs in it
+    for (_, _, filenames) in os.walk(lib_dir):
+        for filename in filenames:
+            if lib_re.match(filename) is not None:
+                found_libs.add(filename)
+
+    if len(found_libs) == 0:
+        print("Could not find any libraries from us")
+        return False
+
+    # This should match up the count and names of libraries installed
+    # vs the build configuration (eg static lib installed or not)
+
+    return True
+
+def verify_includes(build_config):
+    include_dir = build_config['installed_include_dir']
+    if not os.path.isdir(include_dir):
+        print('Error: installed_include_dir "%s" is not a directory' % include_dir)
+        return False
+
+    expected_headers = set(build_config['public_headers'] + build_config['external_headers'])
+    found_headers = set([])
+
+    for (_, _, filenames) in os.walk(include_dir):
+        for filename in filenames:
+            found_headers.add(filename)
+
+    if found_headers != expected_headers:
+        missing = expected_headers - found_headers
+        extra = found_headers - expected_headers
+
+        if len(missing) > 0:
+            print("Missing expected headers: %s" % (" ".join(sorted(missing))))
+
+        if len(extra) > 0:
+            print("Have unexpected headers: %s" % (" ".join(sorted(extra))))
+        return False
+
+    return True
+
+def main(args=None):
+    if args is None:
+        args = sys.argv
+
+    if len(args) < 2:
+        print("Usage: %s <build_config.json>" % args[0])
         return 1
-    install_prefix = sys.argv[1]
+
+    with open(os.path.join(args[1])) as f:
+        build_config = json.load(f)
+
+    install_prefix = build_config['prefix']
 
     if not os.path.isdir(install_prefix):
         print('Error: install_prefix "%s" is not a directory' % install_prefix)
         return 1
 
-    found_libs = False
+    if not verify_includes(build_config):
-    found_headers = False
-
-    for (_, _, filenames) in os.walk(install_prefix):
-        for filename in filenames:
-            if is_header_file(filename):
-                found_headers = True
-            elif is_lib_file(filename):
-                found_libs = True
-        if found_libs and found_headers:
-            return 0
-
-    print("Error: installation incomplete. Found headers: %s. Found libs: %s. install_prefix was %s"
-            % (found_headers, found_libs, install_prefix))
         return 1
 
+    if not verify_library(build_config):
+        return 1
+
+    return 0
 
 if __name__ == '__main__':
     sys.exit(main())