fix(python-bindings): added darwin duplicate rpath patch and fixed python rpath poiting

due to a current bug in meson-python duplicate rpaths are registered in the shared object files created by meson-python. The new masos dynamic loader refuses to load shared object files with duplicate rpaths. There is a small patch script which removes any duplicates. This is a temporary but effective fix (https://github.com/mesonbuild/meson-python/issues/813). Further, there was an issue due to mixed use of pure python and C++ code with name conflicts. This has been resolved so that both python and C++ code can be imported just find now.

build-python/fix_rpaths.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import glob
+import subprocess
+from collections import OrderedDict
+
+def get_rpaths(binary_path):
+    """Uses otool to extract a list of all LC_RPATH entries."""
+    print(f"--- Checking rpaths for: {binary_path}")
+    rpaths = []
+    try:
+        proc = subprocess.run(
+            ['otool', '-l', binary_path],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+
+        lines = proc.stdout.splitlines()
+        for i, line in enumerate(lines):
+            if "cmd LC_RPATH" in line.strip():
+                if i + 2 < len(lines):
+                    path_line = lines[i + 2].strip()
+                    if path_line.startswith("path "):
+                        # Extract the path, e.g., "path /foo/bar (offset 12)"
+                        rpath = path_line.split(" ")[1]
+                        rpaths.append(rpath)
+
+    except (subprocess.CalledProcessError, FileNotFoundError) as e:
+        print(f"Error running otool: {e}")
+        return []
+
+    return rpaths
+
+def fix_rpaths(binary_path):
+    all_rpaths = get_rpaths(binary_path)
+    if not all_rpaths:
+        print("--- No rpaths found or otool failed.")
+        return
+
+    unique_rpaths = list(OrderedDict.fromkeys(all_rpaths))
+    for rpath in unique_rpaths:
+        print(f" - RPATH: {rpath}")
+
+    if len(all_rpaths) == len(unique_rpaths):
+        print("--- No duplicate rpaths found. Nothing to do.")
+        return
+
+    print(f"--- Found {len(all_rpaths)} rpaths; {len(unique_rpaths)} are unique.")
+    print(f"--- Fixing duplicates in: {binary_path}")
+
+    try:
+        for rpath in all_rpaths:
+            subprocess.run(
+                ['install_name_tool', '-delete_rpath', rpath, binary_path],
+                check=True,
+                capture_output=True
+            )
+
+        for rpath in unique_rpaths:
+            subprocess.run(
+                ['install_name_tool', '-add_rpath', rpath, binary_path],
+                check=True,
+                capture_output=True
+            )
+
+        print("--- Successfully fixed rpaths.")
+
+    except (subprocess.CalledProcessError, FileNotFoundError) as e:
+        print(f"--- Error running install_name_tool: {e}")
+        if e.stderr:
+            print(f"STDERR: {e.stderr.decode()}")
+        if e.stdout:
+            print(f"STDOUT: {e.stdout.decode()}")
+        sys.exit(1) # Fail the install if we can't fix it
+
+def main():
+    if len(sys.argv) != 2:
+        print(f"--- Error: Expected one argument (path to .so file), got {sys.argv}", file=sys.stderr)
+        sys.exit(1)
+
+    # Get the file path directly from the command line argument
+    so_file_path = sys.argv[1]
+
+    if not os.path.exists(so_file_path):
+        print(f"--- Error: File not found at {so_file_path}", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"--- Fixing rpaths for built file: {so_file_path}")
+    fix_rpaths(so_file_path)
+
+if __name__ == "__main__":
+    main()

build-python/import_debug.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+SITE=$(python3 -c "import site; print(site.getsitepackages()[0])")
+SO_FILE=$(find "$SITE/fourdst" -name "_phys.cpython-*-darwin.so")
+
+echo "=== File Locations ==="
+echo "Extension module: $SO_FILE"
+echo "Libs directory: $SITE/.fourdst.mesonpy.libs/"
+echo ""
+
+echo "=== RPATH Entries ==="
+otool -l "$SO_FILE" | grep -A 3 LC_RPATH
+echo ""
+
+echo "=== Library Dependencies ==="
+otool -L "$SO_FILE"
+echo ""
+
+echo "=== Checking if dependencies exist ==="
+for lib in $(otool -L "$SO_FILE" | grep -v ":" | awk '{print $1}' | grep -v "^/usr" | grep -v "^/System"); do
+    echo "Checking: $lib"
+    if [[ "$lib" == @* ]]; then
+        resolved=$(echo "$lib" | sed "s|@loader_path|$SITE/fourdst|g")
+        if [ -f "$resolved" ]; then
+            echo "  Found: $resolved"
+        else
+            echo "  NOT FOUND: $resolved"
+        fi
+    fi
+done
+echo ""
+
+echo "=== Attempting import with verbose error ==="
+python3 -c "
+import sys
+sys.path.insert(0, '$SITE')
+try:
+    import fourdst._phys
+    print('Import successful!')
+except ImportError as e:
+    print(f'Import failed: {e}')
+    import traceback
+    traceback.print_exc()
+"
+echo ""
+
+echo "=== Contents of .fourdst.mesonpy.libs ==="
+ls -lh "$SITE/.fourdst.mesonpy.libs/" 2>/dev/null || echo "Directory not found!"

build-python/meson.build

@@ -1,8 +1,8 @@
 # --- Python Extension Setup ---
-py_installation = import('python').find_installation('python3')
+py_installation = import('python').find_installation('python3', pure: false)
 
 py_mod = py_installation.extension_module(
-    'fourdst', # Name of the generated .so/.pyd file (without extension)
+    '_phys', # Name of the generated .so/.pyd file (without extension)
     sources: [
         meson.project_source_root() + '/src-pybind/bindings.cpp',
         meson.project_source_root() + '/src-pybind/composition/bindings.cpp',
@@ -17,97 +17,99 @@ py_mod = py_installation.extension_module(
     ],
     cpp_args : ['-UNDEBUG'],
     install : true,
+    subdir: 'fourdst',
 )
 
 py_installation.install_sources(
-    meson.project_source_root() + '/fourdst/__init__.py',
+    meson.project_source_root() + '/src-pybind/fourdst/__init__.py',
     subdir: 'fourdst',
 )
+
 py_installation.install_sources(
     files(
-        meson.project_source_root() + '/fourdst/cli/__init__.py',
-        meson.project_source_root() + '/fourdst/cli/main.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/__init__.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/main.py',
     ),
     subdir: 'fourdst/cli',
 )
 py_installation.install_sources(
     files(
-        meson.project_source_root() + '/fourdst/cli/templates/meson.build.in',
-        meson.project_source_root() + '/fourdst/cli/templates/plugin.cpp.in',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/templates/meson.build.in',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/templates/plugin.cpp.in',
     ),
     subdir: 'fourdst/cli/templates',
 )
 py_installation.install_sources(
     files(
-        meson.project_source_root() + '/fourdst/cli/bundle/__init__.py',
-        meson.project_source_root() + '/fourdst/cli/bundle/create.py',
-        meson.project_source_root() + '/fourdst/cli/bundle/fill.py',
-        meson.project_source_root() + '/fourdst/cli/bundle/inspect.py',
-        meson.project_source_root() + '/fourdst/cli/bundle/sign.py',
-        meson.project_source_root() + '/fourdst/cli/bundle/clear.py',
-        meson.project_source_root() + '/fourdst/cli/bundle/diff.py',
-        meson.project_source_root() + '/fourdst/cli/bundle/validate.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/bundle/__init__.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/bundle/create.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/bundle/fill.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/bundle/inspect.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/bundle/sign.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/bundle/clear.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/bundle/diff.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/bundle/validate.py',
     ),
     subdir: 'fourdst/cli/bundle'
 )
 py_installation.install_sources(
     files(
-        meson.project_source_root() + '/fourdst/cli/cache/__init__.py',
-        meson.project_source_root() + '/fourdst/cli/cache/clear.py'
+        meson.project_source_root() + '/src-pybind/fourdst/cli/cache/__init__.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/cache/clear.py'
     ),
     subdir: 'fourdst/cli/cache'
 )
 py_installation.install_sources(
     files(
-        meson.project_source_root() + '/fourdst/cli/common/__init__.py',
-        meson.project_source_root() + '/fourdst/cli/common/config.py',
-        meson.project_source_root() + '/fourdst/cli/common/templates.py',
-        meson.project_source_root() + '/fourdst/cli/common/utils.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/common/__init__.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/common/config.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/common/templates.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/common/utils.py',
     ),
     subdir: 'fourdst/cli/common'
 )
 py_installation.install_sources(
     files(
-        meson.project_source_root() + '/fourdst/cli/keys/__init__.py',
-        meson.project_source_root() + '/fourdst/cli/keys/generate.py',
-        meson.project_source_root() + '/fourdst/cli/keys/sync.py',
-        meson.project_source_root() + '/fourdst/cli/keys/add.py',
-        meson.project_source_root() + '/fourdst/cli/keys/list.py',
-        meson.project_source_root() + '/fourdst/cli/keys/remove.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/keys/__init__.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/keys/generate.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/keys/sync.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/keys/add.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/keys/list.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/keys/remove.py',
     ),
     subdir: 'fourdst/cli/keys'
 )
 py_installation.install_sources(
     files(
-        meson.project_source_root() + '/fourdst/cli/keys/remote/__init__.py',
-        meson.project_source_root() + '/fourdst/cli/keys/remote/add.py',
-        meson.project_source_root() + '/fourdst/cli/keys/remote/list.py',
-        meson.project_source_root() + '/fourdst/cli/keys/remote/remove.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/keys/remote/__init__.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/keys/remote/add.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/keys/remote/list.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/keys/remote/remove.py',
     ),
     subdir: 'fourdst/cli/keys/remote'
 )
 py_installation.install_sources(
     files(
-        meson.project_source_root() + '/fourdst/cli/plugin/__init__.py',
-        meson.project_source_root() + '/fourdst/cli/plugin/init.py',
-        meson.project_source_root() + '/fourdst/cli/plugin/pack.py',
-        meson.project_source_root() + '/fourdst/cli/plugin/extract.py',
-        meson.project_source_root() + '/fourdst/cli/plugin/diff.py',
-        meson.project_source_root() + '/fourdst/cli/plugin/validate.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/plugin/__init__.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/plugin/init.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/plugin/pack.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/plugin/extract.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/plugin/diff.py',
+        meson.project_source_root() + '/src-pybind/fourdst/cli/plugin/validate.py',
     ),
     subdir: 'fourdst/cli/plugin'
 )
 
 py_installation.install_sources(
     files(
-        meson.project_source_root() + '/fourdst/core/__init__.py',
-        meson.project_source_root() + '/fourdst/core/build.py',
-        meson.project_source_root() + '/fourdst/core/bundle.py',
-        meson.project_source_root() + '/fourdst/core/config.py',
-        meson.project_source_root() + '/fourdst/core/platform.py',
-        meson.project_source_root() + '/fourdst/core/utils.py',
-        meson.project_source_root() + '/fourdst/core/keys.py',
-        meson.project_source_root() + '/fourdst/core/plugin.py',
+        meson.project_source_root() + '/src-pybind/fourdst/core/__init__.py',
+        meson.project_source_root() + '/src-pybind/fourdst/core/build.py',
+        meson.project_source_root() + '/src-pybind/fourdst/core/bundle.py',
+        meson.project_source_root() + '/src-pybind/fourdst/core/config.py',
+        meson.project_source_root() + '/src-pybind/fourdst/core/platform.py',
+        meson.project_source_root() + '/src-pybind/fourdst/core/utils.py',
+        meson.project_source_root() + '/src-pybind/fourdst/core/keys.py',
+        meson.project_source_root() + '/src-pybind/fourdst/core/plugin.py',
     ),
     subdir: 'fourdst/core'
 )

meson.build

@@ -19,19 +19,19 @@ subdir('src-pybind')
 
 # Bundle the Python backend for the Electron app
 
-if get_option('electron-build-py-backend')
-  pyinstaller_exe = find_program('pyinstaller', required : true)
-  electron_src_dir = meson.current_source_dir() / 'electron'
-
-  custom_target('fourdst-backend',
-    input : electron_src_dir / 'fourdst-backend.spec',
-    # The output is the directory that PyInstaller creates.
-    # We are interested in the executable inside it.
-    output : 'fourdst-backend',
-    # The command to run. We tell PyInstaller where to put the final executable.
-    command : [pyinstaller_exe, '--distpath', meson.current_build_dir() / 'electron/dist', '--workpath', meson.current_build_dir() / 'electron/build', '--noconfirm', '@INPUT@'],
-    # This ensures the backend is built whenever you run 'meson compile'.
-    build_by_default : true
-  )
-endif
+#if get_option('electron-build-py-backend')
+#  pyinstaller_exe = find_program('pyinstaller', required : true)
+#  electron_src_dir = meson.current_source_dir() / 'electron'
+#
+#  custom_target('fourdst-backend',
+#    input : electron_src_dir / 'fourdst-backend.spec',
+#    # The output is the directory that PyInstaller creates.
+#    # We are interested in the executable inside it.
+#    output : 'fourdst-backend',
+#    # The command to run. We tell PyInstaller where to put the final executable.
+#    command : [pyinstaller_exe, '--distpath', meson.current_build_dir() / 'electron/dist', '--workpath', meson.current_build_dir() / 'electron/build', '--noconfirm', '@INPUT@'],
+#    # This ensures the backend is built whenever you run 'meson compile'.
+#    build_by_default : true
+#  )
+#endif
 

pip_install_mac_patch.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+# pip_install_mac.sh - Temporary workaround for meson-python duplicate RPATH bug on macOS
+
+set -e
+
+# Color codes for output
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+
+echo -e "${YELLOW}"
+echo "========================================================================="
+echo "                    TEMPORARY INSTALLATION WORKAROUND"
+echo "========================================================================="
+echo -e "${NC}"
+echo ""
+echo -e "${YELLOW}WARNING:${NC} This script applies a temporary patch to fix a known issue with"
+echo "meson-python that causes duplicate RPATH entries in built Python extensions"
+echo "on macOS, preventing module imports."
+echo ""
+echo "This workaround will:"
+echo "  1. Install fourdst using pip"
+echo "  2. Locate the installed extension module"
+echo "  3. Remove duplicate RPATH entries using install_name_tool"
+echo ""
+echo "This is a temporary solution while the meson-python team resolves the"
+echo "duplicate RPATH bug. For more information, see:"
+echo "  https://github.com/mesonbuild/meson-python/issues/813"
+echo ""
+echo -e "${YELLOW}Do you understand and wish to continue? [y/N]${NC} "
+read -r response
+
+if [[ ! "$response" =~ ^[Yy]$ ]]; then
+    echo -e "${RED}Installation cancelled.${NC}"
+    exit 1
+fi
+
+echo ""
+echo -e "${GREEN}Step 1: Finding current Python environment...${NC}"
+
+# Get the current Python executable
+PYTHON_BIN=$(which python3)
+if [ -z "$PYTHON_BIN" ]; then
+    echo -e "${RED}Error: python3 not found in PATH${NC}"
+    exit 1
+fi
+
+echo "Using Python: $PYTHON_BIN"
+PYTHON_VERSION=$($PYTHON_BIN --version)
+echo "Python version: $PYTHON_VERSION"
+
+# Get site-packages directory
+SITE_PACKAGES=$($PYTHON_BIN -c "import site; print(site.getsitepackages()[0])")
+echo "Site packages: $SITE_PACKAGES"
+echo ""
+
+echo -e "${GREEN}Step 2: Installing fourdst with pip...${NC}"
+$PYTHON_BIN -m pip install . -v
+
+if [ $? -ne 0 ]; then
+    echo -e "${RED}Error: pip install failed${NC}"
+    exit 1
+fi
+echo ""
+
+echo -e "${GREEN}Step 3: Locating installed extension module...${NC}"
+
+# Find the .so file
+SO_FILE=$(find "$SITE_PACKAGES/fourdst" -name "_phys.cpython-*-darwin.so" 2>/dev/null | head -n 1)
+
+if [ -z "$SO_FILE" ]; then
+    echo -e "${RED}Error: Could not find _phys.cpython-*-darwin.so in $SITE_PACKAGES/fourdst${NC}"
+    echo "Installation may have failed or the file is in an unexpected location."
+    exit 1
+fi
+
+echo "Found extension module: $SO_FILE"
+echo ""
+
+echo -e "${GREEN}Step 4: Running RPATH fix script...${NC}"
+
+# Check if fix_rpath.py exists
+FIX_SCRIPT="build-python/fix_rpaths.py"
+if [ ! -f "$FIX_SCRIPT" ]; then
+    echo -e "${RED}Error: $FIX_SCRIPT not found${NC}"
+    echo "Please ensure you're running this script from the project root directory."
+    exit 1
+fi
+
+# Run the fix script
+$PYTHON_BIN "$FIX_SCRIPT" "$SO_FILE"
+
+if [ $? -ne 0 ]; then
+    echo -e "${RED}Error: RPATH fix script failed${NC}"
+    exit 1
+fi
+
+echo ""
+echo -e "${GREEN}=========================================================================${NC}"
+echo -e "${GREEN}                    Installation Complete!${NC}"
+echo -e "${GREEN}=========================================================================${NC}"
+echo ""
+echo "You can now use fourdst in your Python environment."
+echo ""
+echo "Test the installation with:"
+echo "  $PYTHON_BIN -c 'import fourdst; print(fourdst.__version__)'"
+echo ""
+echo -e "${YELLOW}Note:${NC} If you reinstall or upgrade fourdst, you will need to run this"
+echo "script again to apply the RPATH fix."
+echo ""

src-pybind/bindings.cpp

@@ -6,7 +6,7 @@
 #include "composition/bindings.h"
 #include "config/bindings.h"
 
-PYBIND11_MODULE(fourdst, m) {
+PYBIND11_MODULE(_phys, m) {
     m.doc() = "Python bindings for the fourdst utility modules which are a part of the 4D-STAR project.";
 
     auto compMod  = m.def_submodule("composition", "Composition-module bindings");

src-pybind/fourdst/__init__.py

@@ -0,0 +1,11 @@
+from ._phys import *
+import sys
+
+from ._phys import atomic, composition, constants, config
+
+sys.modules['fourdst.atomic'] = atomic
+sys.modules['fourdst.composition'] = composition
+sys.modules['fourdst.constants'] = constants
+sys.modules['fourdst.config'] = config
+
+__all__ = ['atomic', 'composition', 'constants', 'config', 'core', 'cli']