diff --git a/PACKAGE_EXCEPTIONS.md b/PACKAGE_EXCEPTIONS.md new file mode 100644 index 0000000..1426529 --- /dev/null +++ b/PACKAGE_EXCEPTIONS.md @@ -0,0 +1,147 @@ +# Package Exceptions Configuration + +This document explains how to configure package-specific exceptions for different platforms and architectures. + +## Overview + +The build system supports multiple PyPI index URLs and intelligent package handling: + +- **Multiple Index URLs**: Support for additional PyPI repositories via command-line arguments +- **Smart Fallback**: Automatically tries binary wheels first, then falls back to source builds +- **Platform Exceptions**: Configure packages that need special handling for specific platforms + +The build system now supports three types of package exceptions: + +1. **Skip**: Packages that should be completely skipped for specific platforms +2. **Force Source**: Packages that should always be built from source for specific platforms +3. **Platform Specific**: Advanced configuration with custom actions and reasons + +## Configuration File + +Package exceptions are configured in `package-exceptions.json` at the root of the project. + +## Extra Index URLs + +You can specify additional PyPI index URLs when running the build script: + +```bash +# Build with additional index URLs +node ./scripts/build.js 3.12.6 "https://custom.pypi.org/simple/,https://another.pypi.org/simple/" + +# Build with single additional index +node ./scripts/build.js 3.12.6 "https://custom.pypi.org/simple/" +``` + +The build system will: +1. Always include the default NVIDIA PyPI index (`https://pypi.nvidia.com`) +2. Add any additional URLs you specify +3. Search all indexes when downloading packages + +## Configuration Structure + +```json +{ + "skip": { + "package-name": { + "platform-key": "reason for skipping" + } + }, + "forceSource": { + "package-name": { + "platform-key": "reason for forcing source build" + } + }, + "platformSpecific": { + "package-name": { + "platform-key": { + "action": "skip|forceSource", + "reason": "detailed explanation" + } + } + } +} +``` + +## Platform Keys + +Platform keys follow the format: `{os}-{arch}` + +- `linux-x64` - Linux AMD64 +- `linux-aarch64` - Linux ARM64 +- `macosx-x64` - macOS Intel +- `macosx-aarch64` - macOS Apple Silicon +- `windows-x64` - Windows AMD64 + +## Examples + +### Skipping CUDA Packages on Non-Linux Platforms + +```json +{ + "skip": { + "cudf-cu12": { + "macosx-x64": "CUDA packages not supported on macOS", + "macosx-aarch64": "CUDA packages not supported on macOS", + "windows-x64": "CUDA packages not supported on Windows" + } + } +} +``` + +### Forcing Source Build for Packages Without Binary Wheels + +```json +{ + "forceSource": { + "parasail": { + "linux-aarch64": "parasail has no binary wheels for Linux ARM64", + "macosx-aarch64": "parasail has no binary wheels for macOS ARM64" + } + } +} +``` + +### Advanced Platform-Specific Configuration + +```json +{ + "platformSpecific": { + "tensorflow": { + "linux-aarch64": { + "action": "skip", + "reason": "TensorFlow has limited ARM64 support" + }, + "macosx-aarch64": { + "action": "forceSource", + "reason": "TensorFlow ARM64 builds are experimental" + } + } + } +} +``` + +## Adding New Exceptions + +1. Edit `package-exceptions.json` +2. Add your package and platform-specific rules +3. Test the build to ensure the exceptions work as expected +4. Commit the changes + +## Best Practices + +- **Be specific**: Only add exceptions when absolutely necessary +- **Document reasons**: Always provide clear explanations for why exceptions exist +- **Test thoroughly**: Verify that exceptions work on all affected platforms +- **Keep updated**: Remove exceptions when packages add support for new platforms +- **Use simple configs**: Prefer `skip` and `forceSource` over `platformSpecific` when possible + +## Troubleshooting + +If the build fails to load the exceptions configuration: + +1. Check that `package-exceptions.json` is valid JSON +2. Verify the file is in the project root +3. Check file permissions +4. Look for console warnings during build startup + +The build will continue with an empty configuration if the file cannot be loaded. \ No newline at end of file diff --git a/README.md b/README.md index 4551807..4af93b8 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,29 @@ As we do not maintain/build our own java, the version of this package is not bou specific version of python, but newer python version publications produce newer entrypoints in this package. +## Package Management + +The build system automatically handles package installation with intelligent fallback: +- First attempts to download binary wheels from PyPI +- Falls back to building from source if binary wheels are unavailable +- Supports platform-specific exceptions for packages with limited platform support +- Supports additional PyPI index URLs for custom package sources + +### Usage + +```bash +# Basic build +npm run build + +# Build with additional PyPI index URLs +node ./scripts/build.js 3.12.6 "https://custom.pypi.org/simple/,https://another.pypi.org/simple/" + +# Build with single additional index +node ./scripts/build.js 3.12.6 "https://custom.pypi.org/simple/" +``` + +See [PACKAGE_EXCEPTIONS.md](PACKAGE_EXCEPTIONS.md) for details on configuring package-specific behavior. + ## How to release new version of python run environment 1. Update `package.json`: diff --git a/package-exceptions.json b/package-exceptions.json new file mode 100644 index 0000000..2e7c864 --- /dev/null +++ b/package-exceptions.json @@ -0,0 +1,35 @@ +{ + "skip": { + "cudf-cu12": { + "macosx-x64": "CUDA packages not supported on macOS", + "macosx-aarch64": "CUDA packages not supported on macOS", + "windows-x64": "CUDA packages not supported on Windows" + }, + "cupy-cuda12x": { + "macosx-x64": "CUDA packages not supported on macOS", + "macosx-aarch64": "CUDA packages not supported on macOS", + "windows-x64": "CUDA packages not supported on Windows" + } + }, + "forceSource": { + "parasail": { + "linux-aarch64": "parasail has no binary wheels for Linux ARM64", + "macosx-aarch64": "parasail has no binary wheels for macOS ARM64" + }, + "pynacl": { + "windows-x64": "pynacl often fails to build on Windows, prefer source" + } + }, + "platformSpecific": { + "tensorflow": { + "linux-aarch64": { + "action": "skip", + "reason": "TensorFlow has limited ARM64 support" + }, + "macosx-aarch64": { + "action": "forceSource", + "reason": "TensorFlow ARM64 builds are experimental" + } + } + } +} \ No newline at end of file diff --git a/packages.txt b/packages.txt index 5ed5586..3e64f01 100644 --- a/packages.txt +++ b/packages.txt @@ -17,4 +17,4 @@ parasail==1.3.4 numpy==2.2.6 umap-learn==0.5.7 PyYAML==6.0.2 -cudf-cu12==25.4.0 +cudf-cu12==25.6.0 \ No newline at end of file diff --git a/scripts/build.js b/scripts/build.js index 4316c1a..6feb7ff 100755 --- a/scripts/build.js +++ b/scripts/build.js @@ -5,11 +5,16 @@ */ const args = process.argv.slice(2); -if (args.length !== 1) { - console.error(`Usage: ${process.argv[0]} `); +if (args.length < 1 || args.length > 2) { + console.error(`Usage: ${process.argv[0]} [extra-index-urls]`); + console.error(' extra-index-urls: Comma-separated list of additional PyPI index URLs'); + console.error(' Example: node build.js 3.12.6 "https://pypi.org/simple/,https://custom.pypi.org/simple/"'); process.exit(1); } +const version = args[0]; +const extraIndexUrls = args[1] ? args[1].split(',').map(url => url.trim()) : []; + const cp = require('child_process'); const { promisify } = require('util'); const fs = require('fs'); @@ -36,6 +41,24 @@ const os_windows = 'windows'; const arch_x64 = 'x64'; const arch_aarch64 = 'aarch64'; +// Load package exceptions configuration +let PACKAGE_EXCEPTIONS = { + skip: {}, + forceSource: {}, + platformSpecific: {} +}; + +try { + const exceptionsPath = path.join(packageRoot, 'package-exceptions.json'); + if (fs.existsSync(exceptionsPath)) { + PACKAGE_EXCEPTIONS = JSON.parse(fs.readFileSync(exceptionsPath, 'utf-8')); + console.log('Loaded package exceptions configuration'); + } +} catch (error) { + console.warn('Failed to load package exceptions configuration:', error.message); + // Fall back to default empty configuration +} + /* * Function definitions */ @@ -353,7 +376,83 @@ async function consolidateLibsOSX(installDir) { } } -function downloadPackages(pyBin, dependenciesFile, destinationDir, osType, archType) { +function getPackageName(packageSpec) { + // Extract package name from spec (e.g., "parasail==1.3.4" -> "parasail") + return packageSpec.split(/[<>=!]/)[0].trim(); +} + +function shouldSkipPackage(packageName, osType, archType) { + const platformKey = `${osType}-${archType}`; + + // Check simple skip configuration + const skipConfig = PACKAGE_EXCEPTIONS.skip[packageName]; + if (skipConfig && skipConfig[platformKey]) { + console.log(` ⚠️ Skipping ${packageName} for ${platformKey}: ${skipConfig[platformKey]}`); + return true; + } + + // Check platform-specific configuration + const platformConfig = PACKAGE_EXCEPTIONS.platformSpecific[packageName]; + if (platformConfig && platformConfig[platformKey]) { + const action = platformConfig[platformKey].action; + const reason = platformConfig[platformKey].reason; + + if (action === 'skip') { + console.log(` ⚠️ Skipping ${packageName} for ${platformKey}: ${reason}`); + return true; + } + } + + return false; +} + +function shouldForceSource(packageName, osType, archType) { + const platformKey = `${osType}-${archType}`; + + // Check simple forceSource configuration + const forceSourceConfig = PACKAGE_EXCEPTIONS.forceSource[packageName]; + if (forceSourceConfig && forceSourceConfig[platformKey]) { + console.log(` ℹ️ Forcing source build for ${packageName} on ${platformKey}: ${forceSourceConfig[platformKey]}`); + return true; + } + + // Check platform-specific configuration + const platformConfig = PACKAGE_EXCEPTIONS.platformSpecific[packageName]; + if (platformConfig && platformConfig[platformKey]) { + const action = platformConfig[platformKey].action; + const reason = platformConfig[platformKey].reason; + + if (action === 'forceSource') { + console.log(` ℹ️ Forcing source build for ${packageName} on ${platformKey}: ${reason}`); + return true; + } + } + + return false; +} + +function buildPipArgs(packageSpec, destinationDir, extraIndexUrls = []) { + const args = [ + '-m', + 'pip', + 'download', + packageSpec, + '--dest', + destinationDir + ]; + + // Add default NVIDIA index + args.push('--extra-index-url=https://pypi.nvidia.com'); + + // Add additional index URLs + for (const url of extraIndexUrls) { + args.push('--extra-index-url=' + url); + } + + return args; +} + +async function downloadPackages(pyBin, dependenciesFile, destinationDir, osType, archType, extraIndexUrls = []) { const depsContent = fs.readFileSync(dependenciesFile, 'utf-8'); const depsList = depsContent.split('\n'); @@ -364,21 +463,52 @@ function downloadPackages(pyBin, dependenciesFile, destinationDir, osType, archT continue; } - if (archType === arch_aarch64 && depSpecClean.startsWith('parasail')) { + const packageName = getPackageName(depSpecClean); + console.log(`\nProcessing package: ${depSpecClean}`); + + // Check if package should be skipped for this platform + if (shouldSkipPackage(packageName, osType, archType)) { continue; } - - runCommand(pyBin, [ - '-m', - 'pip', - 'download', - '--extra-index-url=https://pypi.nvidia.com', - depSpec.trim(), - '--only-binary', - ':all:', - '--dest', - destinationDir - ]); + + // Check if package should be forced to build from source + const forceSource = shouldForceSource(packageName, osType, archType); + + if (forceSource) { + // Skip binary wheel attempt and go straight to source + console.log(` Building from source (forced)...`); + try { + const pipArgs = buildPipArgs(depSpecClean, destinationDir, extraIndexUrls); + pipArgs.push('--no-binary', ':all:'); + runCommand(pyBin, pipArgs); + console.log(` ✓ Successfully downloaded source for ${depSpecClean}`); + } catch (sourceError) { + console.error(` ✗ Failed to download source for ${depSpecClean}: ${sourceError.message}`); + throw sourceError; + } + } else { + // Try binary wheel first, then fall back to source + try { + console.log(` Attempting to download binary wheel...`); + const pipArgs = buildPipArgs(depSpecClean, destinationDir, extraIndexUrls); + pipArgs.push('--only-binary', ':all:'); + runCommand(pyBin, pipArgs); + console.log(` ✓ Successfully downloaded binary wheel for ${depSpecClean}`); + } catch (error) { + console.log(` ✗ Binary wheel not available for ${depSpecClean}, building from source...`); + + // If binary wheel download fails, build from source + try { + const pipArgs = buildPipArgs(depSpecClean, destinationDir, extraIndexUrls); + pipArgs.push('--no-binary', ':all:'); + runCommand(pyBin, pipArgs); + console.log(` ✓ Successfully downloaded source for ${depSpecClean}`); + } catch (sourceError) { + console.error(` ✗ Failed to download source for ${depSpecClean}: ${sourceError.message}`); + throw sourceError; + } + } + } } } @@ -406,7 +536,6 @@ function copyDirSync(src, dest) { */ (async () => { - const version = args[0]; const osType = currentOS(); const archType = currentArch(); const installDir = path.join( @@ -434,6 +563,10 @@ function copyDirSync(src, dest) { const packagesDir = path.join(installDir, 'packages'); const dependenciesFile = path.join(packageRoot, 'packages.txt'); - downloadPackages(pyBin, dependenciesFile, packagesDir, osType, archType); + if (extraIndexUrls.length > 0) { + console.log(`\nUsing additional PyPI index URLs: ${extraIndexUrls.join(', ')}`); + } + + await downloadPackages(pyBin, dependenciesFile, packagesDir, osType, archType, extraIndexUrls); runCommand('pl-pkg', ['build', 'packages', `--package-id=${version}`]); })();