Make: Better builds for shared libraries (#136)

* Fixed build issues with the shared libraries

* Fixed arm shared library march

* Added missing defined check

* Made the mem* exported using `#pragma` instead of from cmake

* Changed non-mask AVX512 loads/stores using epi8  to `si512`

* Fixed python build 64bit platform detection

* Changed python MSVC build all AVX functions

* Removed explicit arm arch

* Fixed python build to properly detect platform when cross-compiling

* Avoid including `arm_acle` & `arm_sve` for MSVC

* Update clang version to 16 for cross compile builds

* Added check for `cibuildwheels` in python builds

* Removed cmake dependency for python builds

Author: ashbob999

.github/workflows/prerelease.yml

@@ -202,11 +202,11 @@ jobs:
     name: Cross Compilation
     runs-on: ubuntu-22.04
     env:
-      CC: clang
-      CXX: clang++
-      AR: llvm-ar
-      NM: llvm-nm
-      RANLIB: llvm-ranlib
+      CC: clang-16
+      CXX: clang++-16
+      AR: llvm-ar-16
+      NM: llvm-nm-16
+      RANLIB: llvm-ranlib-16
 
     strategy:
       fail-fast: false
@@ -222,11 +222,15 @@ jobs:
 
         # C/C++
         # We need to install the cross-compilation toolchain for ARM64 and ARMHF
+        # Clang 16 isn't available from default repos on Ubuntu 22.04, so we have to install it manually
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y clang lld make crossbuild-essential-arm64 crossbuild-essential-armhf
-
+          sudo apt-get install -y make build-essential crossbuild-essential-arm64 crossbuild-essential-armhf  libjemalloc-dev
+          wget https://apt.llvm.org/llvm.sh
+          chmod +x llvm.sh
+          sudo ./llvm.sh 16
+        
       - name: Build C/C++
         run: |
           cmake -B build_artifacts \

CMakeLists.txt

@@ -38,6 +38,14 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
     "MinSizeRel" "RelWithDebInfo")
 endif()
 
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64|amd64")
+  SET(SZ_PLATFORM_X86 TRUE)
+  message(STATUS "Platform: x86")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64")
+  SET(SZ_PLATFORM_ARM TRUE)
+  message(STATUS "Platform: ARM")
+endif()
+
 # Determine if StringZilla is built as a subproject (using `add_subdirectory`)
 # or if it is the main project
 set(STRINGZILLA_IS_MAIN_PROJECT OFF)
@@ -99,8 +107,17 @@ if(${CMAKE_VERSION} VERSION_EQUAL 3.13 OR ${CMAKE_VERSION} VERSION_GREATER 3.13)
   enable_testing()
 endif()
 
+if (MSVC)
+  # Remove /RTC* from MSVC debug flags by default (it will be added back in the set_compiler_flags function)
+  # Beacuse /RTC* cannot be used without the crt so it needs to be disabled for that specifc target
+  string(REGEX REPLACE "/RTC[^ ]*" "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
+  string(REGEX REPLACE "/RTC[^ ]*" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
+endif()
+
 # Function to set compiler-specific flags
 function(set_compiler_flags target cpp_standard target_arch)
+  get_target_property(target_type ${target} TYPE)
+
   target_include_directories(${target} PRIVATE scripts)
   target_link_libraries(${target} PRIVATE ${STRINGZILLA_TARGET_NAME})
 
@@ -152,17 +169,26 @@ function(set_compiler_flags target cpp_standard target_arch)
     "$<$<AND:$<CXX_COMPILER_ID:MSVC>,$<OR:$<CONFIG:Debug>,$<CONFIG:RelWithDebInfo>>>:/Zi>"
   )
 
+  if(NOT target_type STREQUAL "SHARED_LIBRARY")
+    if(MSVC)
+      target_compile_options(${target} PRIVATE "$<$<CONFIG:Debug>:/RTC1>")
+    endif()
+  endif()
+
   # If available, enable Position Independent Code
-  if(CMAKE_POSITION_INDEPENDENT_CODE)
+  get_target_property(target_pic ${target} POSITION_INDEPENDENT_CODE)
+  if(target_pic)
     target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-fPIC>")
     target_link_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-fPIC>")
+    target_compile_definitions(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:SZ_PIC>")
   endif()
 
   # Avoid builtin functions where we know what we are doing.
   target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-fno-builtin-memcmp>")
   target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-fno-builtin-memchr>")
   target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-fno-builtin-memcpy>")
   target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-fno-builtin-memset>")
+  target_compile_options(${target} PRIVATE "$<$<CXX_COMPILER_ID:MSVC>:/Oi->")
 
   # Check for ${target_arch} and set it or use the current system if not defined
   if("${target_arch}" STREQUAL "")
@@ -202,17 +228,19 @@ function(set_compiler_flags target cpp_standard target_arch)
 
   # Sanitizer options for Debug mode
   if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-    target_compile_options(
-      ${target}
-      PRIVATE
-      "$<$<CXX_COMPILER_ID:GNU,Clang>:-fsanitize=address;-fsanitize=leak>"
-      "$<$<CXX_COMPILER_ID:MSVC>:/fsanitize=address>")
-
-    target_link_options(
-      ${target}
-      PRIVATE
-      "$<$<CXX_COMPILER_ID:GNU,Clang>:-fsanitize=address;-fsanitize=leak>"
-      "$<$<CXX_COMPILER_ID:MSVC>:/fsanitize=address>")
+    if(NOT target_type STREQUAL "SHARED_LIBRARY")
+      target_compile_options(
+        ${target}
+        PRIVATE
+        "$<$<CXX_COMPILER_ID:GNU,Clang>:-fsanitize=address;-fsanitize=leak>"
+        "$<$<CXX_COMPILER_ID:MSVC>:/fsanitize=address>")
+
+      target_link_options(
+        ${target}
+        PRIVATE
+        "$<$<CXX_COMPILER_ID:GNU,Clang>:-fsanitize=address;-fsanitize=leak>"
+        "$<$<CXX_COMPILER_ID:MSVC>:/fsanitize=address>")
+    endif()
 
     # Define SZ_DEBUG macro based on build configuration
     target_compile_definitions(
@@ -248,7 +276,7 @@ if(${STRINGZILLA_BUILD_TEST})
 
   # Check system architecture to avoid complex cross-compilation workflows, but
   # compile multiple backends: disabling all SIMD, enabling only AVX2, only AVX-512, only Arm Neon.
-  if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64|amd64")
+  if(SZ_PLATFORM_X86)
     # x86 specific backends
     if (MSVC)
       define_launcher(stringzilla_test_cpp20_x86_serial scripts/test.cpp 20 "AVX")
@@ -259,37 +287,60 @@ if(${STRINGZILLA_BUILD_TEST})
       define_launcher(stringzilla_test_cpp20_x86_avx2 scripts/test.cpp 20 "haswell")
       define_launcher(stringzilla_test_cpp20_x86_avx512 scripts/test.cpp 20 "sapphirerapids")
     endif()
-  elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64")
+  elseif(SZ_PLATFORM_ARM)
     # ARM specific backends
     define_launcher(stringzilla_test_cpp20_arm_serial scripts/test.cpp 20 "armv8-a")
     define_launcher(stringzilla_test_cpp20_arm_neon scripts/test.cpp 20 "armv8-a+simd")
   endif()
 endif()
 
 if(${STRINGZILLA_BUILD_SHARED})
-  add_library(stringzilla_shared SHARED c/lib.c)
-  set_compiler_flags(stringzilla_shared "" "${STRINGZILLA_TARGET_ARCH}")
+
+  function(define_shared target)
+    add_library(${target} SHARED c/lib.c)
+
+    set_target_properties(${target} PROPERTIES
+      VERSION ${PROJECT_VERSION}
+      SOVERSION 1
+      POSITION_INDEPENDENT_CODE ON
+      PUBLIC_HEADER include/stringzilla/stringzilla.h)
+
+    if (SZ_PLATFORM_X86)
+      if (MSVC)
+        set_compiler_flags(${target} "" "SSE2")
+      else()
+        set_compiler_flags(${target} "" "ivybridge")
+      endif()
+
+      target_compile_definitions(${target} PRIVATE
+        "SZ_USE_X86_AVX512=1"
+        "SZ_USE_X86_AVX2=1"
+        "SZ_USE_ARM_NEON=0"
+        "SZ_USE_ARM_SVE=0")
+    elseif(SZ_PLATFORM_ARM)
+      set_compiler_flags(${target} "" "armv8-a")
+
+      target_compile_definitions(${target} PRIVATE
+        "SZ_USE_X86_AVX512=0"
+        "SZ_USE_X86_AVX2=0"
+        "SZ_USE_ARM_NEON=1"
+        "SZ_USE_ARM_SVE=1")
+    endif()
+  endfunction()
+
+  define_shared(stringzilla_shared)
   target_compile_definitions(stringzilla_shared PRIVATE "SZ_AVOID_LIBC=0")
   target_compile_definitions(stringzilla_shared PRIVATE "SZ_OVERRIDE_LIBC=1")
-  set_target_properties(stringzilla_shared PROPERTIES
-  VERSION ${PROJECT_VERSION}
-  SOVERSION 1
-  POSITION_INDEPENDENT_CODE ON
-  PUBLIC_HEADER include/stringzilla/stringzilla.h)
-  
+
   # Try compiling a version without linking the LibC
-  add_library(stringzillite SHARED c/lib.c)
-  set_compiler_flags(stringzillite "" "${STRINGZILLA_TARGET_ARCH}")
+  define_shared(stringzillite)
   target_compile_definitions(stringzillite PRIVATE "SZ_AVOID_LIBC=1")
   target_compile_definitions(stringzillite PRIVATE "SZ_OVERRIDE_LIBC=1")
-  set_target_properties(stringzillite PROPERTIES
-    VERSION ${PROJECT_VERSION}
-    SOVERSION 1
-    POSITION_INDEPENDENT_CODE ON
-    PUBLIC_HEADER include/stringzilla/stringzilla.h)
 
   # Avoid built-ins on MSVC and other compilers, as that will cause compileration errors
   target_compile_options(stringzillite PRIVATE
     "$<$<CXX_COMPILER_ID:GNU,Clang>:-fno-builtin;-nostdlib>"
-    "$<$<CXX_COMPILER_ID:MSVC>:/Oi->")
-endif()
+    "$<$<CXX_COMPILER_ID:MSVC>:/Oi-;/GS->")
+  target_link_options(stringzillite PRIVATE "$<$<CXX_COMPILER_ID:GNU,Clang>:-nostdlib>")
+  target_link_options(stringzillite PRIVATE "$<$<CXX_COMPILER_ID:MSVC>:/NODEFAULTLIB>")
+endif()

c/lib.c

@@ -224,6 +224,14 @@ BOOL WINAPI DllMain(HINSTANCE hints, DWORD forward_reason, LPVOID lp) {
     case DLL_PROCESS_DETACH: return TRUE;
     }
 }
+
+#if SZ_AVOID_LIBC
+BOOL WINAPI _DllMainCRTStartup(HINSTANCE hints, DWORD forward_reason, LPVOID lp) {
+    DllMain(hints, forward_reason, lp);
+    return TRUE;
+}
+#endif
+
 #else
 __attribute__((constructor)) static void sz_dispatch_table_init_on_gcc_or_clang(void) { sz_dispatch_table_init(); }
 #endif
@@ -356,30 +364,54 @@ SZ_DYNAMIC void sz_generate(sz_cptr_t alphabet, sz_size_t alphabet_size, sz_ptr_
     sz_generate_serial(alphabet, alphabet_size, result, result_length, generator, generator_user_data);
 }
 
-// It's much harder to override the C standard library on Windows and MSVC,
-// so we'll just provide the symbols for other Operating Systems.
-#if SZ_OVERRIDE_LIBC && !(defined(_WIN32) || defined(__CYGWIN__))
+// Provide overrides for the libc mem* functions
+#if SZ_OVERRIDE_LIBC && !(defined(__CYGWIN__))
 
+// SZ_DYNAMIC can't be use here for MSVC, because MSVC complains about different linkage (C2375), probably due to to the
+// CRT headers specifying the function as __declspec(dllimport), there might be a combination of defines that works. But
+// for now they will be manually exported using linker flags
+
+#if defined(_MSC_VER)
+#pragma comment(linker, "/export:memchr")
+void *__cdecl memchr(void const *s, int c_wide, size_t n) {
+#else
 SZ_DYNAMIC void *memchr(void const *s, int c_wide, size_t n) {
+#endif
     sz_u8_t c = (sz_u8_t)c_wide;
     return (void *)sz_find_byte(s, n, (sz_cptr_t)&c);
 }
 
+#if defined(_MSC_VER)
+#pragma comment(linker, "/export:memcpy")
+void *__cdecl memcpy(void *dest, void const *src, size_t n) {
+#else
 SZ_DYNAMIC void *memcpy(void *dest, void const *src, size_t n) {
+#endif
     sz_copy(dest, src, n);
     return (void *)dest;
 }
 
+#if defined(_MSC_VER)
+#pragma comment(linker, "/export:memmove")
+void *__cdecl memmove(void *dest, void const *src, size_t n) {
+#else
 SZ_DYNAMIC void *memmove(void *dest, void const *src, size_t n) {
+#endif
     sz_move(dest, src, n);
     return (void *)dest;
 }
 
+#if defined(_MSC_VER)
+#pragma comment(linker, "/export:memset")
+void *__cdecl memset(void *s, int c, size_t n) {
+#else
 SZ_DYNAMIC void *memset(void *s, int c, size_t n) {
+#endif
     sz_fill(s, n, c);
     return (void *)s;
 }
 
+#if !defined(_MSC_VER)
 SZ_DYNAMIC void *memmem(void const *h, size_t h_len, void const *n, size_t n_len) {
     return (void *)sz_find(h, h_len, n, n_len);
 }
@@ -393,5 +425,5 @@ SZ_DYNAMIC void memfrob(void *s, size_t n) {
     char const *base64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
     sz_generate(base64, 64, s, n, SZ_NULL, SZ_NULL);
 }
-
+#endif
 #endif // SZ_OVERRIDE_LIBC