Build static librdkafka_windows.a with travis and mingw (@neptoess, #3130)

* Build static librdkafka_windows.a with travis * Update paths to match Travis MinGW build script * Update DLL paths * Replace tabs with spaces * Don't use msys2 rm, just use regular travis one * Add Win32 DLLs to MinGW C++ pkg-config * Relocate MinGW target_link_libraries to allow building executables * Update CMakeLists.txt to include pkg-config for Windows DLLs for static MinGW builds * Write to librdkafka.a instead of librdkafka_windows.a Update CMakeLists.txt to only generate -static .pc files when building static lib Run cmake-format on CMakeLists.txt * Update packing script for different .a file name * Separate MinGW static build and dynamic build Turn WITH_SASL=ON for Windows build so OATHBEARER SASL can be used * Restore previous whitespace style in CMakeLists.txt * CMakeLists.txt formatting * Undo CMakeLists.txt whitespace-only changes Remove cmake options where we like the defaults Add libdl and libz to static MinGW build to support WITH_PLUGINS and WITH_ZLIB * Fix accidental tab characters in packaging/mingw-w64/travis-before-install.sh * Default WITH_PLUGINS to ON on Windows CMake builds Remove libdl from MinGW build Fix typo when WITHOUT_WIN32_CONFIG is ON * Modify CMAKE_FIND_LIBRARY_SUFFIXES instead of deleting .dll.a files * Make use of Libs.private * Fix a CMake check using WITH_PLUGINS when it should have been using WITH_LIBDL * Add strip calls for MinGW static libraries * Slight CMakeLists.txt refactor per SpaceIm comment * Move CMAKE_FIND_LIBRARY_SUFFIXES from CMakeLists.txt to the Travis build script Co-authored-by: neptoess <neptoess@gmail.com>
2021-04-07 02:30:24 -04:00 · 2021-04-07 02:30:24 -04:00 · 2a10b46d42
--- a/.travis.yml
+++ b/.travis.yml
@ -53,7 +53,7 @@ matrix:
   env: LINKAGE=static HOMEBREW_NO_AUTO_UPDATE=1
   before_script:
    - ./configure --install-deps --disable-lz4-ext --prefix="$PWD/dest" --enable-static --disable-syslog --enable-strip
- - name: "Windows MinGW-w64"
+ - name: "Windows MinGW-w64 Dynamic"
   os: windows
   env:
    - SKIP_MAKE=y
@ -61,6 +61,14 @@ matrix:
    - source ./packaging/mingw-w64/travis-before-install.sh
   before_script:
    - ./packaging/mingw-w64/configure-build-msys2-mingw.sh
+ - name: "Windows MinGW-w64 Static"
+   os: windows
+   env:
+    - SKIP_MAKE=y
+   before_install:
+    - source ./packaging/mingw-w64/travis-before-install.sh
+   before_script:
+    - ./packaging/mingw-w64/configure-build-msys2-mingw-static.sh
 - name: "Linux GCC: +integration-tests +copyright-check +doc-check"
   os: linux
   dist: xenial
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -79,7 +79,7 @@ endif()
 # }

 # WITH_PLUGINS {
-if(WITH_LIBDL)
+if(WITH_LIBDL OR WIN32)
  set(with_plugins_default ON)
 else()
  set(with_plugins_default OFF)
@ -261,8 +261,6 @@ install(
    DESTINATION "share/licenses/librdkafka"
 )

-# }
-
 add_subdirectory(src)
 add_subdirectory(src-cpp)

--- a/packaging/mingw-w64/configure-build-msys2-mingw-static.sh
+++ b/packaging/mingw-w64/configure-build-msys2-mingw-static.sh
@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+
+cmake \
+    -G "MinGW Makefiles" \
+    -D CMAKE_FIND_LIBRARY_SUFFIXES=".a" \
+    -D CMAKE_INSTALL_PREFIX="$PWD/dest/" \
+    -D RDKAFKA_BUILD_STATIC=ON \
+    .
+
+$mingw64 mingw32-make
+$mingw64 mingw32-make install
+
+# Bundle all the static dependencies with the static lib we just built
+mkdir mergescratch
+pushd mergescratch
+cp /C/tools/msys64/mingw64/lib/libzstd.a ./
+cp /C/tools/msys64/mingw64/lib/libcrypto.a ./
+cp /C/tools/msys64/mingw64/lib/liblz4.a ./
+cp /C/tools/msys64/mingw64/lib/libssl.a ./
+cp /C/tools/msys64/mingw64/lib/libz.a ./
+cp ../src/librdkafka.a ./
+
+# Have to rename because ar won't work with + in the name
+cp ../src-cpp/librdkafka++.a ./librdkafkacpp.a
+ar -M << EOF
+create librdkafka-static.a
+addlib librdkafka.a
+addlib libzstd.a
+addlib libcrypto.a
+addlib liblz4.a
+addlib libssl.a
+addlib libz.a
+save
+end
+EOF
+
+ar -M << EOF
+create librdkafkacpp-static.a
+addlib librdkafka-static.a
+addlib librdkafkacpp.a
+save
+end
+EOF
+
+strip -g ./librdkafka-static.a
+strip -g ./librdkafkacpp-static.a
+cp ./librdkafka-static.a ../dest/lib/
+cp ./librdkafkacpp-static.a ../dest/lib/librdkafka++-static.a
+popd
+rm -rf ./mergescratch
+
+export PATH="$PWD/dest/bin:/mingw64/bin/:${PATH}"
+cd tests
+./test-runner.exe -l -Q -p1
--- a/packaging/mingw-w64/travis-before-install.sh
+++ b/packaging/mingw-w64/travis-before-install.sh
@ -20,9 +20,10 @@ case $TRAVIS_OS_NAME in
        # to zstd instead of xz compression
        $msys2 pacman -Sy --noconfirm pacman
        choco upgrade --no-progress -y msys2
-        $msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake mingw-w64-x86_64-openssl mingw-w64-x86_64-cyrus-sasl

        ## Install more MSYS2 packages from https://packages.msys2.org/base here
+        $msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-gcc mingw-w64-x86_64-make mingw-w64-x86_64-cmake mingw-w64-x86_64-openssl mingw-w64-x86_64-lz4 mingw-w64-x86_64-zstd
+
        taskkill //IM gpg-agent.exe //F  || true  # https://travis-ci.community/t/4967
        export PATH=/C/tools/msys64/mingw64/bin:$PATH
        export MAKE=mingw32-make  # so that Autotools can find it
--- a/packaging/nuget/packaging.py
+++ b/packaging/nuget/packaging.py
@ -512,6 +512,10 @@ class StaticPackage (Package):
            # osx static lib and pkg-config file
            [{'arch': 'x64', 'plat': 'osx', 'fname_glob': 'librdkafka-clang.tar.gz'}, './lib/librdkafka-static.a', 'librdkafka_darwin.a'],
            [{'arch': 'x64', 'plat': 'osx', 'fname_glob': 'librdkafka-clang.tar.gz'}, './lib/pkgconfig/rdkafka-static.pc', 'librdkafka_darwin.pc'],
+
+            # win static lib and pkg-config file
+            [{'arch': 'x64', 'plat': 'win', 'fname_glob': 'librdkafka-mingw.tar.gz'}, './lib/librdkafka-static.a', 'librdkafka_windows.a'],
+            [{'arch': 'x64', 'plat': 'win', 'fname_glob': 'librdkafka-mingw.tar.gz'}, './lib/pkgconfig/rdkafka-static.pc', 'librdkafka_windows.pc'],
        ]

        for m in mappings:
@ -577,7 +581,9 @@ class StaticPackage (Package):
            "./librdkafka_musl_linux.a",
            "./librdkafka_musl_linux.pc",
            "./librdkafka_darwin.a",
-            "./librdkafka_darwin.pc"]
+            "./librdkafka_darwin.pc",
+            "./librdkafka_windows.a",
+            "./librdkafka_windows.pc"]

        missing = list()
        with zfile.ZFile(path, 'r') as zf:
--- a/src-cpp/CMakeLists.txt
+++ b/src-cpp/CMakeLists.txt
@ -37,49 +37,40 @@ if(NOT RDKAFKA_BUILD_STATIC)
 endif()

 # Generate pkg-config file
-set(PKG_CONFIG_NAME
-    "librdkafka++"
-)
-set(PKG_CONFIG_DESCRIPTION
-    "The Apache Kafka C/C++ library"
-)
-set(PKG_CONFIG_VERSION
-    "${PROJECT_VERSION}"
-)
-set(PKG_CONFIG_REQUIRES "rdkafka")
-set(PKG_CONFIG_CFLAGS
-    "-I\${includedir}"
-)
-set(PKG_CONFIG_LIBS
-    "-L\${libdir} -lrdkafka++"
-)
-set(PKG_CONFIG_LIBS_PRIVATE
-    "-lrdkafka"
-)
-configure_file(
+set(PKG_CONFIG_VERSION "${PROJECT_VERSION}")
+if(NOT RDKAFKA_BUILD_STATIC)
+  set(PKG_CONFIG_NAME "librdkafka++")
+  set(PKG_CONFIG_DESCRIPTION "The Apache Kafka C/C++ library")
+  set(PKG_CONFIG_REQUIRES "rdkafka")
+  set(PKG_CONFIG_CFLAGS "-I\${includedir}")
+  set(PKG_CONFIG_LIBS "-L\${libdir} -lrdkafka++")
+  set(PKG_CONFIG_LIBS_PRIVATE "-lrdkafka")
+  configure_file(
    "../packaging/cmake/rdkafka.pc.in"
    "${GENERATED_DIR}/rdkafka++.pc"
    @ONLY
-)
-install(FILES ${GENERATED_DIR}/rdkafka++.pc
+  )
+  install(
+    FILES ${GENERATED_DIR}/rdkafka++.pc
    DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig"
-)
-if(RDKAFKA_BUILD_STATIC)
-  set(PKG_CONFIG_NAME
-    "librdkafka++-static"
-  )
-  set(PKG_CONFIG_DESCRIPTION
-    "The Apache Kafka C/C++ library (static)"
-  )
-  set(PKG_CONFIG_LIBS
-    "-L\${libdir} \${libdir}/librdkafka++.a"
  )
+else()
+  set(PKG_CONFIG_NAME "librdkafka++-static")
+  set(PKG_CONFIG_DESCRIPTION "The Apache Kafka C/C++ library (static)")
+  set(PKG_CONFIG_REQUIRES "")
+  set(PKG_CONFIG_CFLAGS "-I\${includedir} -DLIBRDKAFKA_STATICLIB")
+  set(PKG_CONFIG_LIBS "-L\${libdir} \${libdir}/librdkafka++.a")
+  if(WIN32)
+    set(PKG_CONFIG_LIBS_PRIVATE "-lws2_32 -lsecur32 -lcrypt32")
+  endif()
+
  configure_file(
    "../packaging/cmake/rdkafka.pc.in"
    "${GENERATED_DIR}/rdkafka++-static.pc"
    @ONLY
  )
-  install(FILES ${GENERATED_DIR}/rdkafka.pc
+  install(
+    FILES ${GENERATED_DIR}/rdkafka++-static.pc
    DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig"
  )
 endif()
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@ -152,9 +152,9 @@ if(WITHOUT_WIN32_CONFIG)
    list(APPEND rdkafka_compile_definitions ENABLE_DEVEL=0)
  endif(ENABLE_DEVEL)
  if(WITH_PLUGINS)
-    list(APPEND rdkafka_compile_definitions WITH_PLUGIN=1)
+    list(APPEND rdkafka_compile_definitions WITH_PLUGINS=1)
  else()
-    list(APPEND rdkafka_compile_definitions WITH_PLUGIN=0)
+    list(APPEND rdkafka_compile_definitions WITH_PLUGINS=0)
  endif(WITH_PLUGINS)
 endif()

@ -182,20 +182,6 @@ if(RDKAFKA_BUILD_STATIC)
  target_compile_definitions(rdkafka PUBLIC LIBRDKAFKA_STATICLIB)
 endif()

-if(MINGW)
-	message(STATUS "Detected MinGW build")
-  # MinGW ignores `#pragma comment(lib, ...)` so we need to specify link libs
-  target_link_libraries(rdkafka PUBLIC crypt32 ws2_32 secur32)
-endif(MINGW)
-
-if(WIN32)  
-  if(RDKAFKA_BUILD_STATIC)
-    target_link_libraries(rdkafka PUBLIC crypt32)
-  else()
-    target_compile_definitions(rdkafka PRIVATE LIBRDKAFKA_EXPORTS)
-  endif()
-endif()
-
 # We need 'dummy' directory to support `#include "../config.h"` path
 set(dummy "${GENERATED_DIR}/dummy")
 file(MAKE_DIRECTORY "${dummy}")
@ -257,80 +243,85 @@ if(WITH_LZ4_EXT)
  target_link_libraries(rdkafka PUBLIC LZ4::LZ4)
 endif()

-# Set up path to these sources for other sub-projects (tests, examples)
-# to be able to reach them.
-#set(rdkafka_SRC_DIR ${PROJECT_SOURCE_DIR}
-#    CACHE INTERNAL "${PROJECT_NAME} source dir" FORCE)
+if(WIN32)
+  if(WITH_SSL)
+    target_link_libraries(rdkafka PUBLIC crypt32)
+  endif()
+
+  target_link_libraries(rdkafka PUBLIC ws2_32 secur32)
+  if(NOT RDKAFKA_BUILD_STATIC)
+    target_compile_definitions(rdkafka PRIVATE LIBRDKAFKA_EXPORTS)
+  endif()
+endif()

 # Generate pkg-config file
-set(PKG_CONFIG_NAME
-    "librdkafka"
-)
-set(PKG_CONFIG_DESCRIPTION
-    "The Apache Kafka C/C++ library"
-)
-set(PKG_CONFIG_VERSION
-    "${PROJECT_VERSION}"
-)
+set(PKG_CONFIG_VERSION "${PROJECT_VERSION}")
 set(PKG_CONFIG_REQUIRES "")
-if(WITH_ZLIB)
-  string(APPEND PKG_CONFIG_REQUIRES "zlib ")
+if (WIN32)
+  set(PKG_CONFIG_LIBS_PRIVATE "-lws2_32 -lsecur32 -lcrypt32")
+else()
+  set(PKG_CONFIG_LIBS_PRIVATE "-lpthread")
+  find_library(RT_LIBRARY rt)
+  if(RT_LIBRARY)
+    string(APPEND PKG_CONFIG_LIBS_PRIVATE " -lrt")
+  endif()
+
+  if(WITH_LIBDL)
+    string(APPEND PKG_CONFIG_LIBS_PRIVATE " -ldl")
+  endif()
+
+  if(WITH_HDRHISTOGRAM)
+    string(APPEND PKG_CONFIG_LIBS_PRIVATE " -lm")
+  endif()
 endif()
-if(WITH_SSL)
+
+if(NOT RDKAFKA_BUILD_STATIC)
+  set(PKG_CONFIG_NAME "librdkafka")
+  set(PKG_CONFIG_DESCRIPTION "The Apache Kafka C/C++ library")
+  if(WITH_ZLIB)
+    string(APPEND PKG_CONFIG_REQUIRES "zlib ")
+  endif()
+
+  if(WITH_SSL)
    string(APPEND PKG_CONFIG_REQUIRES "libssl ")
-endif()
-if(WITH_SASL_CYRUS)
-  string(APPEND PKG_CONFIG_REQUIRES "libsasl2 ")
-endif()
-if(WITH_ZSTD)
+  endif()
+
+  if(WITH_SASL_CYRUS)
+    string(APPEND PKG_CONFIG_REQUIRES "libsasl2 ")
+  endif()
+
+  if(WITH_ZSTD)
    string(APPEND PKG_CONFIG_REQUIRES "libzstd ")
-endif()
-if(WITH_LZ4_EXT)
+  endif()
+
+  if(WITH_LZ4_EXT)
    string(APPEND PKG_CONFIG_REQUIRES "liblz4 ")
-endif()
-set(PKG_CONFIG_CFLAGS
-    "-I\${includedir}"
-)
-set(PKG_CONFIG_LIBS
-    "-L\${libdir} -lrdkafka"
-)
-set(PKG_CONFIG_LIBS_PRIVATE
-    "-lpthread"
-)
-find_library(RT_LIBRARY rt)
-if(RT_LIBRARY)
-  string(APPEND PKG_CONFIG_LIBS_PRIVATE " -lrt")
-endif()
-if(WITH_PLUGINS)
-  string(APPEND PKG_CONFIG_LIBS_PRIVATE " -ldl")
-endif()
-if(WITH_HDRHISTOGRAM)
-  string(APPEND PKG_CONFIG_LIBS_PRIVATE " -lm")
-endif()
-configure_file(
-  "../packaging/cmake/rdkafka.pc.in"
-  "${GENERATED_DIR}/rdkafka.pc"
-  @ONLY
-)
-install(FILES ${GENERATED_DIR}/rdkafka.pc
-  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig"
-)
-if(RDKAFKA_BUILD_STATIC)
-  set(PKG_CONFIG_NAME
-    "librdkafka-static"
+  endif()
+
+  set(PKG_CONFIG_CFLAGS "-I\${includedir}")
+  set(PKG_CONFIG_LIBS "-L\${libdir} -lrdkafka")
+
+  configure_file(
+    "../packaging/cmake/rdkafka.pc.in"
+    "${GENERATED_DIR}/rdkafka.pc"
+    @ONLY
  )
-  set(PKG_CONFIG_DESCRIPTION
-    "The Apache Kafka C/C++ library (static)"
-  )
-  set(PKG_CONFIG_LIBS
-    "-L\${libdir} \${libdir}/librdkafka.a"
+  install(
+    FILES ${GENERATED_DIR}/rdkafka.pc
+    DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig"
  )
+else()
+  set(PKG_CONFIG_NAME "librdkafka-static")
+  set(PKG_CONFIG_DESCRIPTION "The Apache Kafka C/C++ library (static)")
+  set(PKG_CONFIG_CFLAGS "-I\${includedir} -DLIBRDKAFKA_STATICLIB")
+  set(PKG_CONFIG_LIBS "-L\${libdir} \${libdir}/librdkafka.a")
  configure_file(
    "../packaging/cmake/rdkafka.pc.in"
    "${GENERATED_DIR}/rdkafka-static.pc"
    @ONLY
  )
-  install(FILES ${GENERATED_DIR}/rdkafka.pc
+  install(
+    FILES ${GENERATED_DIR}/rdkafka-static.pc
    DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig"
  )
 endif()