fix: CI build deps and Conan GLIBC compatibility

Keep --update in conan install so stale recipe revisions in the persistent
CI Conan cache get refreshed, but force bison/flex/libtool/openssl to build
from source on Linux to avoid remote binaries compiled against GLIBC_2.29+.

Install missing deps in ci/entrypoint.sh (openblas-devel for cblas.h,
perl-IPC-Cmd/perl-Digest-SHA for openssl source build) until the base
image is rebuilt from scripts/Dockerfile.manylinux.*.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: MrLoh

ci/entrypoint.sh

@@ -1,5 +1,17 @@
 #!/bin/bash
-git config --global --add safe.directory /root/milvus-lite/thirdparty/milvus;
+set -e
+git config --global --add safe.directory /root/milvus-lite/thirdparty/milvus
+
+# Deps missing from the current milvusdb/milvus-env:lite-main image:
+#  - openblas-devel: cblas.h header needed by knowhere
+#  - libatomic-static: static libatomic.a needed at link time
+#  - perl-IPC-Cmd/perl-Digest-SHA: needed when openssl is built from source via Conan
+# These can be removed once the base image is rebuilt from scripts/Dockerfile.manylinux.*
+yum install -y openblas-devel libatomic-static perl-IPC-Cmd perl-Digest-SHA 2>/dev/null || true
+cp /usr/lib/gcc/x86_64-redhat-linux/8/libatomic.a /usr/lib64/ 2>/dev/null || true
+# RHEL 8 openblas-devel puts headers in /usr/include/openblas/ not /usr/include/
+ln -sf /usr/include/openblas/*.h /usr/include/ 2>/dev/null || true
+
 cd python
 python3 -m build --wheel
 

python/setup.py

@@ -105,11 +105,16 @@ def run(self):
             subprocess.call(['conan', 'remote', 'add', 'default-conan-local', 'https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local'],
                                   cwd=build_temp, env=env)
             if sys.platform.lower() == 'linux':
-                subprocess.check_call(['conan', 'install', extdir, '--build=missing', '-s', 'build_type=Release', '-s', 'compiler.libcxx=libstdc++11'],
+                subprocess.check_call(['conan', 'install', extdir,
+                                       '--build=missing', '--update',
+                                       '--build=bison', '--build=flex',
+                                       '--build=libtool', '--build=openssl',
+                                       '-s', 'build_type=Release',
+                                       '-s', 'compiler.libcxx=libstdc++11'],
                                       cwd=build_temp, env=env)
             else:
                 # macos
-                subprocess.check_call(['conan', 'install', extdir, '--build=missing', '-s', 'build_type=Release'], cwd=build_temp, env=env)
+                subprocess.check_call(['conan', 'install', extdir, '--build=missing', '--update', '-s', 'build_type=Release'], cwd=build_temp, env=env)
         # build
         extra_cmake_args = shlex.split(env.get('CMAKE_ARGS', ''))
         extra_build_args = shlex.split(env.get('BUILD_ARGS', ''))

scripts/Dockerfile.manylinux.aarch64

@@ -12,7 +12,7 @@
 
 FROM quay.io/pypa/manylinux_2_28_aarch64:2024.07.07-1
 
-RUN  yum install -y openblas-devel libatomic-static git && cp /usr/lib/gcc/aarch64-redhat-linux/8/libatomic.a /usr/lib64/
+RUN  yum install -y openblas-devel libatomic-static git perl-IPC-Cmd perl-Digest-SHA && cp /usr/lib/gcc/aarch64-redhat-linux/8/libatomic.a /usr/lib64/
 ENV PATH="/opt/_internal/cpython-3.8.19/bin:${PATH}"
 RUN pip3 install conan==1.63.0
 RUN curl https://sh.rustup.rs -sSf | bash -s -- --default-toolchain=1.89 -y

scripts/Dockerfile.manylinux.x86_64

@@ -12,7 +12,7 @@
 
 FROM quay.io/pypa/manylinux_2_28_x86_64:2024.07.07-1
 
-RUN  yum install -y openblas-devel libatomic-static git && cp /usr/lib/gcc/x86_64-redhat-linux/8/libatomic.a /usr/lib64/
+RUN  yum install -y openblas-devel libatomic-static git perl-IPC-Cmd perl-Digest-SHA && cp /usr/lib/gcc/x86_64-redhat-linux/8/libatomic.a /usr/lib64/
 ENV PATH="/opt/_internal/cpython-3.8.19/bin:${PATH}"
 RUN pip3 install conan==1.63.0
 RUN curl https://sh.rustup.rs -sSf | bash -s -- --default-toolchain=1.89 -y

scripts/build_local.sh

@@ -64,9 +64,6 @@ conan remote add default-conan-local \
     https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local \
     --insert 0 >> $LOG 2>&1 || true
 
-# Perl modules needed if openssl must be built from source
-yum install -y perl-IPC-Cmd perl-Digest-SHA >> $LOG 2>&1 || true
-
 cd /workspace/milvus-lite/python
 echo "==> Starting wheel build..." | tee -a $LOG
 python3 -m build --wheel --no-isolation >> $LOG 2>&1 \