|
- import subprocess
- import os
- import os.path
- import errno
- import re
- import shutil
- import sys
- import fnmatch
- import errno
- import platform
- import glob
-
- from contextlib import contextmanager
- from setuptools import Command
- from setuptools import setup, Distribution, Extension
- from setuptools.command.install import install as InstallCommandBase
- from setuptools.command.egg_info import egg_info
-
-
- class BinaryDistribution(Distribution):
- def has_ext_modules(foo):
- return True
-
- RC = 0
-
- ext_name = '.dll' if os.name == 'nt' else ('.dylib' if sys.platform == 'darwin' else '.so')
-
- def git_commit():
- try:
- cmd = ['git', 'rev-parse', 'HEAD']
- git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE,
- cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
- except:
- git_commit = 'Unknown'
- git_commit = git_commit.decode()
- return str(git_commit)
-
- def _get_version_detail(idx):
- assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \
- so detail index must less than 3"
-
- if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'):
- version_details = '@PADDLE_VERSION@'.split('.')
-
- if len(version_details) >= 3:
- return version_details[idx]
-
- return 0
-
- def get_major():
- return int(_get_version_detail(0))
-
- def get_minor():
- return int(_get_version_detail(1))
-
- def get_patch():
- return str(_get_version_detail(2))
-
- def get_cuda_version():
- if '@WITH_GPU@' == 'ON':
- return '@CUDA_VERSION@'
- else:
- return 'False'
-
- def get_cudnn_version():
- if '@WITH_GPU@' == 'ON':
- temp_cudnn_version = ''
- if '@CUDNN_MAJOR_VERSION@':
- temp_cudnn_version += '@CUDNN_MAJOR_VERSION@'
- if '@CUDNN_MINOR_VERSION@':
- temp_cudnn_version += '.@CUDNN_MINOR_VERSION@'
- if '@CUDNN_PATCHLEVEL_VERSION@':
- temp_cudnn_version += '.@CUDNN_PATCHLEVEL_VERSION@'
- return temp_cudnn_version
- else:
- return 'False'
-
- def get_xpu_version():
- if '@WITH_XPU@' == 'ON':
- return '@XPU_BASE_DATE@'
- else:
- return 'False'
-
- def get_xpu_xccl_version():
- if '@WITH_XPU_BKCL@' == 'ON':
- return '@XPU_XCCL_BASE_VERSION@'
- else:
- return 'False'
-
- def is_taged():
- try:
- cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD', '2>/dev/null']
- git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd="@PADDLE_SOURCE_DIR@").communicate()[0].strip()
- git_tag = git_tag.decode()
- except:
- return False
-
- if str(git_tag).replace('v', '') == '@PADDLE_VERSION@':
- return True
- else:
- return False
-
- def write_version_py(filename='paddle/version/__init__.py'):
- cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
- #
- full_version = '%(major)d.%(minor)d.%(patch)s'
- major = '%(major)d'
- minor = '%(minor)d'
- patch = '%(patch)s'
- rc = '%(rc)d'
- cuda_version = '%(cuda)s'
- cudnn_version = '%(cudnn)s'
- xpu_version = '%(xpu)s'
- xpu_xccl_version = '%(xpu_xccl)s'
- istaged = %(istaged)s
- commit = '%(commit)s'
- with_mkl = '%(with_mkl)s'
-
- __all__ = ['cuda', 'cudnn', 'show', 'xpu', 'xpu_xccl']
-
- def show():
- """Get the version of paddle if `paddle` package if tagged. Otherwise, output the corresponding commit id.
-
- Returns:
- If paddle package is not tagged, the commit-id of paddle will be output.
- Otherwise, the following information will be output.
-
- full_version: version of paddle
-
- major: the major version of paddle
-
- minor: the minor version of paddle
-
- patch: the patch level version of paddle
-
- rc: whether it's rc version
-
- cuda: the cuda version of package. It will return `False` if CPU version paddle package is installed
-
- cudnn: the cudnn version of package. It will return `False` if CPU version paddle package is installed
-
- xpu: the xpu version of package. It will return `False` if non-XPU version paddle package is installed
-
- xpu_xccl: the xpu xccl version of package. It will return `False` if non-XPU version paddle package is installed
-
- Examples:
- .. code-block:: python
-
- import paddle
-
- # Case 1: paddle is tagged with 2.2.0
- paddle.version.show()
- # full_version: 2.2.0
- # major: 2
- # minor: 2
- # patch: 0
- # rc: 0
- # cuda: '10.2'
- # cudnn: '7.6.5'
- # xpu: '20230114'
- # xpu_xccl: '1.0.7'
-
- # Case 2: paddle is not tagged
- paddle.version.show()
- # commit: cfa357e984bfd2ffa16820e354020529df434f7d
- # cuda: '10.2'
- # cudnn: '7.6.5'
- # xpu: '20230114'
- # xpu_xccl: '1.0.7'
- """
- if istaged:
- print('full_version:', full_version)
- print('major:', major)
- print('minor:', minor)
- print('patch:', patch)
- print('rc:', rc)
- else:
- print('commit:', commit)
- print('cuda:', cuda_version)
- print('cudnn:', cudnn_version)
- print('xpu:', xpu_version)
- print('xpu_xccl:', xpu_xccl_version)
-
- def mkl():
- return with_mkl
-
- def cuda():
- """Get cuda version of paddle package.
-
- Returns:
- string: Return the version information of cuda. If paddle package is CPU version, it will return False.
-
- Examples:
- .. code-block:: python
-
- import paddle
-
- paddle.version.cuda()
- # '10.2'
-
- """
- return cuda_version
-
- def cudnn():
- """Get cudnn version of paddle package.
-
- Returns:
- string: Return the version information of cudnn. If paddle package is CPU version, it will return False.
-
- Examples:
- .. code-block:: python
-
- import paddle
-
- paddle.version.cudnn()
- # '7.6.5'
-
- """
- return cudnn_version
-
- def xpu():
- """Get xpu version of paddle package.
-
- Returns:
- string: Return the version information of xpu. If paddle package is non-XPU version, it will return False.
-
- Examples:
- .. code-block:: python
-
- import paddle
-
- paddle.version.xpu()
- # '20230114'
-
- """
- return xpu_version
-
- def xpu_xccl():
- """Get xpu xccl version of paddle package.
-
- Returns:
- string: Return the version information of xpu xccl. If paddle package is non-XPU version, it will return False.
-
- Examples:
- .. code-block:: python
-
- import paddle
-
- paddle.version.xpu_xccl()
- # '1.0.7'
-
- """
- return xpu_xccl_version
- '''
- commit = git_commit()
-
- dirname = os.path.dirname(filename)
-
- try:
- os.makedirs(dirname)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
-
- with open(filename, 'w') as f:
- f.write(cnt % {
- 'major': get_major(),
- 'minor': get_minor(),
- 'patch': get_patch(),
- 'rc': RC,
- 'version': '${PADDLE_VERSION}',
- 'cuda': get_cuda_version(),
- 'cudnn': get_cudnn_version(),
- 'xpu': get_xpu_version(),
- 'xpu_xccl': get_xpu_xccl_version(),
- 'commit': commit,
- 'istaged': is_taged(),
- 'with_mkl': '@WITH_MKL@'})
-
- write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version/__init__.py')
-
- def write_cuda_env_config_py(filename='paddle/cuda_env.py'):
- cnt = ""
- if '${JIT_RELEASE_WHL}' == 'ON':
- cnt = '''# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
- #
- import os
- os.environ['CUDA_CACHE_MAXSIZE'] = '805306368'
- '''
-
- with open(filename, 'w') as f:
- f.write(cnt)
-
- write_cuda_env_config_py(filename='@PADDLE_BINARY_DIR@/python/paddle/cuda_env.py')
-
- def write_distributed_training_mode_py(filename='paddle/incubate/distributed/fleet/parameter_server/version.py'):
- cnt = '''
-
- # THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
-
- from paddle.incubate.distributed.fleet.base import Mode
-
- BUILD_MODE=Mode.%(mode)s
-
- def is_transpiler():
- return Mode.TRANSPILER == BUILD_MODE
-
- '''
-
- dirname = os.path.dirname(filename)
-
- try:
- os.makedirs(dirname)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
-
- with open(filename, 'w') as f:
- f.write(cnt % {
- 'mode': 'PSLIB' if '${WITH_PSLIB}' == 'ON' else 'TRANSPILER'
- })
-
- write_distributed_training_mode_py(filename='@PADDLE_BINARY_DIR@/python/paddle/incubate/distributed/fleet/parameter_server/version.py')
-
-
- packages=['paddle',
- 'paddle.libs',
- 'paddle.utils',
- 'paddle.utils.gast',
- 'paddle.utils.cpp_extension',
- 'paddle.dataset',
- 'paddle.reader',
- 'paddle.distributed',
- 'paddle.distributed.communication',
- 'paddle.distributed.communication.stream',
- 'paddle.distributed.metric',
- 'paddle.distributed.ps',
- 'paddle.distributed.ps.utils',
- 'paddle.incubate',
- 'paddle.incubate.autograd',
- 'paddle.incubate.optimizer',
- 'paddle.incubate.checkpoint',
- 'paddle.incubate.operators',
- 'paddle.incubate.tensor',
- 'paddle.incubate.multiprocessing',
- 'paddle.incubate.nn',
- 'paddle.incubate.asp',
- 'paddle.incubate.passes',
- 'paddle.distribution',
- 'paddle.distributed.utils',
- 'paddle.distributed.sharding',
- 'paddle.distributed.fleet',
- 'paddle.distributed.launch',
- 'paddle.distributed.launch.context',
- 'paddle.distributed.launch.controllers',
- 'paddle.distributed.launch.job',
- 'paddle.distributed.launch.plugins',
- 'paddle.distributed.launch.utils',
- 'paddle.distributed.fleet.base',
- 'paddle.distributed.fleet.recompute',
- 'paddle.distributed.fleet.elastic',
- 'paddle.distributed.fleet.meta_optimizers',
- 'paddle.distributed.fleet.meta_optimizers.sharding',
- 'paddle.distributed.fleet.meta_optimizers.ascend',
- 'paddle.distributed.fleet.meta_optimizers.dygraph_optimizer',
- 'paddle.distributed.fleet.runtime',
- 'paddle.distributed.rpc',
- 'paddle.distributed.fleet.dataset',
- 'paddle.distributed.fleet.data_generator',
- 'paddle.distributed.fleet.metrics',
- 'paddle.distributed.fleet.proto',
- 'paddle.distributed.fleet.utils',
- 'paddle.distributed.fleet.layers',
- 'paddle.distributed.fleet.layers.mpu',
- 'paddle.distributed.fleet.meta_parallel',
- 'paddle.distributed.fleet.meta_parallel.pp_utils',
- 'paddle.distributed.fleet.meta_parallel.sharding',
- 'paddle.distributed.fleet.meta_parallel.parallel_layers',
- 'paddle.distributed.auto_parallel',
- 'paddle.distributed.auto_parallel.operators',
- 'paddle.distributed.auto_parallel.tuner',
- 'paddle.distributed.auto_parallel.cost',
- 'paddle.distributed.passes',
- 'paddle.distributed.models',
- 'paddle.distributed.models.moe',
- 'paddle.distributed.transpiler',
- 'paddle.distributed.transpiler.details',
- 'paddle.framework',
- 'paddle.jit',
- 'paddle.jit.dy2static',
- 'paddle.inference',
- 'paddle.inference.contrib',
- 'paddle.inference.contrib.utils',
- 'paddle.fluid',
- 'paddle.fluid.dygraph',
- 'paddle.fluid.proto',
- 'paddle.fluid.proto.profiler',
- 'paddle.fluid.layers',
- 'paddle.fluid.dataloader',
- 'paddle.fluid.contrib',
- 'paddle.fluid.contrib.extend_optimizer',
- 'paddle.fluid.contrib.layers',
- 'paddle.fluid.incubate',
- 'paddle.incubate.distributed.fleet',
- 'paddle.fluid.incubate.checkpoint',
- 'paddle.amp',
- 'paddle.cost_model',
- 'paddle.hapi',
- 'paddle.vision',
- 'paddle.vision.models',
- 'paddle.vision.transforms',
- 'paddle.vision.datasets',
- 'paddle.audio',
- 'paddle.audio.functional',
- 'paddle.audio.features',
- 'paddle.audio.datasets',
- 'paddle.audio.backends',
- 'paddle.text',
- 'paddle.text.datasets',
- 'paddle.incubate',
- 'paddle.incubate.nn',
- 'paddle.incubate.nn.functional',
- 'paddle.incubate.nn.layer',
- 'paddle.incubate.optimizer.functional',
- 'paddle.incubate.autograd',
- 'paddle.incubate.distributed',
- 'paddle.incubate.distributed.utils',
- 'paddle.incubate.distributed.utils.io',
- 'paddle.incubate.distributed.fleet',
- 'paddle.incubate.distributed.models',
- 'paddle.incubate.distributed.models.moe',
- 'paddle.incubate.distributed.models.moe.gate',
- 'paddle.incubate.distributed.fleet.parameter_server',
- 'paddle.incubate.distributed.fleet.parameter_server.distribute_transpiler',
- 'paddle.incubate.distributed.fleet.parameter_server.pslib',
- 'paddle.incubate.distributed.fleet.parameter_server.ir',
- 'paddle.quantization',
- 'paddle.quantization.quanters',
- 'paddle.quantization.observers',
- 'paddle.sparse',
- 'paddle.sparse.nn',
- 'paddle.sparse.nn.layer',
- 'paddle.sparse.nn.functional',
- 'paddle.incubate.xpu',
- 'paddle.io',
- 'paddle.optimizer',
- 'paddle.nn',
- 'paddle.nn.functional',
- 'paddle.nn.layer',
- 'paddle.nn.quant',
- 'paddle.nn.quant.qat',
- 'paddle.nn.initializer',
- 'paddle.nn.utils',
- 'paddle.metric',
- 'paddle.static',
- 'paddle.static.nn',
- 'paddle.static.amp',
- 'paddle.static.amp.bf16',
- 'paddle.static.quantization',
- 'paddle.quantization',
- 'paddle.quantization.imperative',
- 'paddle.tensor',
- 'paddle.onnx',
- 'paddle.autograd',
- 'paddle.device',
- 'paddle.device.cuda',
- 'paddle.device.xpu',
- 'paddle.version',
- 'paddle.profiler',
- 'paddle.geometric',
- 'paddle.geometric.message_passing',
- 'paddle.geometric.sampling',
- ]
-
- with open('@PADDLE_SOURCE_DIR@/python/requirements.txt') as f:
- setup_requires = f.read().splitlines()
-
- if sys.version_info < (3,7):
- raise RuntimeError("Paddle only support Python version>=3.7 now")
-
- if sys.version_info >= (3,7):
- setup_requires_tmp = []
- for setup_requires_i in setup_requires:
- if "<\"3.6\"" in setup_requires_i or "<=\"3.6\"" in setup_requires_i or "<\"3.5\"" in setup_requires_i or "<=\"3.5\"" in setup_requires_i or "<\"3.7\"" in setup_requires_i:
- continue
- setup_requires_tmp+=[setup_requires_i]
- setup_requires = setup_requires_tmp
-
- # the prefix is sys.prefix which should always be usr
- paddle_bins = ''
-
- if not '${WIN32}':
- paddle_bins = ['${PADDLE_BINARY_DIR}/paddle/scripts/paddle']
-
- if os.name != 'nt':
- package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + '.so']}
- else:
- package_data={'paddle.fluid': ['${FLUID_CORE_NAME}' + '.pyd', '${FLUID_CORE_NAME}' + '.lib']}
-
- package_data['paddle.fluid'] += ['${PADDLE_BINARY_DIR}/python/paddle/cost_model/static_op_benchmark.json']
-
- package_dir={
- '': '${PADDLE_BINARY_DIR}/python',
- # The paddle.fluid.proto will be generated while compiling.
- # So that package points to other directory.
- 'paddle.fluid.proto.profiler': '${PADDLE_BINARY_DIR}/paddle/fluid/platform',
- 'paddle.fluid.proto': '${PADDLE_BINARY_DIR}/paddle/fluid/framework',
- 'paddle.fluid': '${PADDLE_BINARY_DIR}/python/paddle/fluid',
- }
-
- # put all thirdparty libraries in paddle.libs
- libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
-
- package_data['paddle.libs']= []
- package_data['paddle.libs']=[
- ('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name,
- ('libwarprnnt' if os.name != 'nt' else 'warprnnt') + ext_name,
- ]
- shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
- shutil.copy('${WARPRNNT_LIBRARIES}', libs_path)
-
- package_data['paddle.libs']+=[
- os.path.basename('${LAPACK_LIB}'),
- os.path.basename('${BLAS_LIB}'),
- os.path.basename('${GFORTRAN_LIB}'),
- os.path.basename('${GNU_RT_LIB_1}')]
- shutil.copy('${BLAS_LIB}', libs_path)
- shutil.copy('${LAPACK_LIB}', libs_path)
- shutil.copy('${GFORTRAN_LIB}', libs_path)
- shutil.copy('${GNU_RT_LIB_1}', libs_path)
- if '${WITH_CUDNN_DSO}' == 'ON' and os.path.exists('${CUDNN_LIBRARY}'):
- package_data['paddle.libs']+=[os.path.basename('${CUDNN_LIBRARY}')]
- shutil.copy('${CUDNN_LIBRARY}', libs_path)
- if sys.platform.startswith("linux") and '${CUDNN_MAJOR_VERSION}' == '8':
- # libcudnn.so includes libcudnn_ops_infer.so, libcudnn_ops_train.so,
- # libcudnn_cnn_infer.so, libcudnn_cnn_train.so, libcudnn_adv_infer.so,
- # libcudnn_adv_train.so
- cudnn_lib_files = glob.glob(os.path.dirname('${CUDNN_LIBRARY}') + '/libcudnn_*so.8')
- for cudnn_lib in cudnn_lib_files:
- if os.path.exists(cudnn_lib):
- package_data['paddle.libs']+=[os.path.basename(cudnn_lib)]
- shutil.copy(cudnn_lib, libs_path)
-
- if not sys.platform.startswith("linux"):
- package_data['paddle.libs']+=[os.path.basename('${GNU_RT_LIB_2}')]
- shutil.copy('${GNU_RT_LIB_2}', libs_path)
-
- if len('${FLASHATTN_LIBRARIES}') > 1:
- package_data['paddle.libs']+=[os.path.basename('${FLASHATTN_LIBRARIES}')]
- shutil.copy('${FLASHATTN_LIBRARIES}', libs_path)
-
- if '${WITH_MKL}' == 'ON':
- shutil.copy('${MKLML_SHARED_LIB}', libs_path)
- shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path)
- package_data['paddle.libs']+=[('libmklml_intel' if os.name != 'nt' else 'mklml') + ext_name, ('libiomp5' if os.name != 'nt' else 'libiomp5md') + ext_name]
- else:
- if os.name == 'nt':
- # copy the openblas.dll
- shutil.copy('${OPENBLAS_SHARED_LIB}', libs_path)
- package_data['paddle.libs'] += ['openblas' + ext_name]
- elif os.name == 'posix' and platform.machine() == 'aarch64' and '${OPENBLAS_LIB}'.endswith('so'):
- # copy the libopenblas.so on linux+aarch64
- # special: libpaddle.so without avx depends on 'libopenblas.so.0', not 'libopenblas.so'
- if os.path.exists('${OPENBLAS_LIB}' + '.0'):
- shutil.copy('${OPENBLAS_LIB}' + '.0', libs_path)
- package_data['paddle.libs'] += ['libopenblas.so.0']
-
- if '${WITH_LITE}' == 'ON':
- shutil.copy('${LITE_SHARED_LIB}', libs_path)
- package_data['paddle.libs']+=['libpaddle_full_api_shared' + ext_name]
- if '${LITE_WITH_NNADAPTER}' == 'ON':
- shutil.copy('${LITE_NNADAPTER_LIB}', libs_path)
- package_data['paddle.libs']+=['libnnadapter' + ext_name]
- if '${NNADAPTER_WITH_HUAWEI_ASCEND_NPU}' == 'ON':
- shutil.copy('${LITE_NNADAPTER_NPU_LIB}', libs_path)
- package_data['paddle.libs']+=['libnnadapter_driver_huawei_ascend_npu' + ext_name]
-
- if '${WITH_CINN}' == 'ON':
- shutil.copy('${CINN_LIB_LOCATION}/${CINN_LIB_NAME}', libs_path)
- shutil.copy('${CINN_INCLUDE_DIR}/cinn/runtime/cuda/cinn_cuda_runtime_source.cuh', libs_path)
- package_data['paddle.libs']+=['libcinnapi.so']
- package_data['paddle.libs']+=['cinn_cuda_runtime_source.cuh']
-
- cinn_fp16_file = '${CINN_INCLUDE_DIR}/cinn/runtime/cuda/float16.h'
- if os.path.exists(cinn_fp16_file):
- shutil.copy(cinn_fp16_file, libs_path)
- package_data['paddle.libs']+=['float16.h']
-
- if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt':
- command = "patchelf --set-rpath '$ORIGIN/' %s/${CINN_LIB_NAME}" % libs_path
- if os.system(command) != 0:
- raise Exception("patch %s/${CINN_LIB_NAME} failed, command: %s" % (libs_path, command))
-
- if '${WITH_PSLIB}' == 'ON':
- shutil.copy('${PSLIB_LIB}', libs_path)
- shutil.copy('${JVM_LIB}', libs_path)
- if os.path.exists('${PSLIB_VERSION_PY}'):
- shutil.copy('${PSLIB_VERSION_PY}', '${PADDLE_BINARY_DIR}/python/paddle/incubate/distributed/fleet/parameter_server/pslib/')
- package_data['paddle.libs'] += ['libps' + ext_name]
- package_data['paddle.libs'] += ['libjvm' + ext_name]
-
- if '${WITH_MKLDNN}' == 'ON':
- if '${CMAKE_BUILD_TYPE}' == 'Release' and os.name != 'nt':
- # only change rpath in Release mode.
- # TODO(typhoonzero): use install_name_tool to patch mkl libs once
- # we can support mkl on mac.
- #
- # change rpath of libdnnl.so.1, add $ORIGIN/ to it.
- # The reason is that all thirdparty libraries in the same directory,
- # thus, libdnnl.so.1 will find libmklml_intel.so and libiomp5.so.
- command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
- if os.system(command) != 0:
- raise Exception("patch libdnnl.so failed, command: %s" % command)
- shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
- if os.name != 'nt':
- shutil.copy('${MKLDNN_SHARED_LIB_1}', libs_path)
- shutil.copy('${MKLDNN_SHARED_LIB_2}', libs_path)
- package_data['paddle.libs']+=['libmkldnn.so.0', 'libdnnl.so.1', 'libdnnl.so.2']
- else:
- package_data['paddle.libs']+=['mkldnn.dll']
-
- if '${WITH_ONNXRUNTIME}' == 'ON':
- shutil.copy('${ONNXRUNTIME_SHARED_LIB}', libs_path)
- shutil.copy('${PADDLE2ONNX_LIB}', libs_path)
- if os.name == 'nt':
- package_data['paddle.libs']+=['paddle2onnx.dll', 'onnxruntime.dll']
- else:
- package_data['paddle.libs']+=['${PADDLE2ONNX_LIB_NAME}', '${ONNXRUNTIME_LIB_NAME}']
-
- if '${WITH_XPU}' == 'ON':
- # only change rpath in Release mode,
- if '${CMAKE_BUILD_TYPE}' == 'Release':
- if os.name != 'nt':
- if "@APPLE@" == "1":
- command = "install_name_tool -id \"@loader_path/\" ${XPU_API_LIB}"
- else:
- command = "patchelf --set-rpath '$ORIGIN/' ${XPU_API_LIB}"
- if os.system(command) != 0:
- raise Exception("patch ${XPU_API_LIB} failed, command: %s" % command)
- shutil.copy('${XPU_API_LIB}', libs_path)
- package_data['paddle.libs']+=['${XPU_API_LIB_NAME}']
- xpu_rt_lib_list = glob.glob('${XPU_RT_LIB}*')
- for xpu_rt_lib_file in xpu_rt_lib_list:
- shutil.copy(xpu_rt_lib_file, libs_path)
- package_data['paddle.libs']+=[os.path.basename(xpu_rt_lib_file)]
-
- if '${WITH_XPU_BKCL}' == 'ON':
- shutil.copy('${XPU_BKCL_LIB}', libs_path)
- package_data['paddle.libs']+=['${XPU_BKCL_LIB_NAME}']
-
- if '${WITH_XPU_XFT}' == 'ON':
- shutil.copy('${XPU_XFT_LIB}', libs_path)
- package_data['paddle.libs']+=['${XPU_XFT_LIB_NAME}']
-
- # remove unused paddle/libs/__init__.py
- if os.path.isfile(libs_path+'/__init__.py'):
- os.remove(libs_path+'/__init__.py')
- package_dir['paddle.libs']=libs_path
-
-
- # change rpath of ${FLUID_CORE_NAME}.ext, add $ORIGIN/../libs/ to it.
- # The reason is that libwarpctc.ext, libiomp5.ext etc are in paddle.libs, and
- # ${FLUID_CORE_NAME}.ext is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
- # This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
- if '${CMAKE_BUILD_TYPE}' == 'Release':
- if os.name != 'nt':
- # only change rpath in Release mode, since in Debug mode, ${FLUID_CORE_NAME}.xx is too large to be changed.
- if "@APPLE@" == "1":
- commands = ["install_name_tool -id '@loader_path/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so']
- commands.append("install_name_tool -add_rpath '@loader_path/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so')
- else:
- commands = ["patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/${FLUID_CORE_NAME}" + '.so']
- # The sw_64 not suppot patchelf, so we just disable that.
- if platform.machine() != 'sw_64' and platform.machine() != 'mips64':
- for command in commands:
- if os.system(command) != 0:
- raise Exception("patch ${FLUID_CORE_NAME}.%s failed, command: %s" % (ext_name, command))
-
- ext_modules = [Extension('_foo', ['stub.cc'])]
- if os.name == 'nt':
- # fix the path separator under windows
- fix_package_dir = {}
- for k, v in package_dir.items():
- fix_package_dir[k] = v.replace('/', '\\')
- package_dir = fix_package_dir
- ext_modules = []
- elif sys.platform == 'darwin':
- ext_modules = []
-
- def find_files(pattern, root, recursive=False):
- for dirpath, _, files in os.walk(root):
- for filename in fnmatch.filter(files, pattern):
- yield os.path.join(dirpath, filename)
- if not recursive:
- break
-
- headers = (
- # paddle level api headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle')) +
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/api')) + # phi unify api header
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/api/ext')) + # custom op api
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/api/include')) + # phi api
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/common')) + # phi common headers
- # phi level api headers (low level api)
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi')) + # phi extension header
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/include', recursive=True)) + # phi include headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/backends', recursive=True)) + # phi backends headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/core', recursive=True)) + # phi core headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/infermeta', recursive=True)) + # phi infermeta headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/kernels')) + # phi kernels headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/kernels/sparse')) + # phi sparse kernels headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/kernels/selected_rows')) + # phi selected_rows kernels headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/kernels/strings')) + # phi sparse kernels headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/kernels/primitive')) + # phi kernel primitive api headers
- # capi headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/phi/capi', recursive=True)) + # phi capi headers
- # profiler headers
- list(find_files('trace_event.h', '@PADDLE_SOURCE_DIR@/paddle/fluid/platform/profiler')) + # phi profiler headers
- # utils api headers
- list(find_files('*.h', '@PADDLE_SOURCE_DIR@/paddle/utils', recursive=True))) # paddle utils headers
-
- jit_layer_headers = ['layer.h', 'serializer.h', 'serializer_utils.h', 'all.h', 'function.h']
- for f in jit_layer_headers:
- headers += list(find_files(f, '@PADDLE_SOURCE_DIR@/paddle/fluid/jit', recursive=True))
-
- if '${WITH_MKLDNN}' == 'ON':
- headers += list(find_files('*', '${MKLDNN_INSTALL_DIR}/include')) # mkldnn
-
- if '${WITH_GPU}' == 'ON' or '${WITH_ROCM}' == 'ON':
- # externalErrorMsg.pb for External Error message
- headers += list(find_files('*.pb', '${externalError_INCLUDE_DIR}'))
-
- if '${WITH_XDNN_API}' == 'ON':
- headers += list(find_files('*.h', '@PADDLE_BINARY_DIR@/third_party/xpu/src/extern_xpu/xpu', recursive=True)) # xdnn api headers
-
- headers += list(find_files('*.h', '${PYBIND_INCLUDE_DIR}', True)) # pybind headers
-
- class InstallCommand(InstallCommandBase):
- def finalize_options(self):
- ret = InstallCommandBase.finalize_options(self)
- self.install_lib = self.install_platlib
- self.install_headers = os.path.join(self.install_platlib, 'paddle', 'include')
- return ret
-
- class InstallHeaders(Command):
- """Override how headers are copied.
- """
- description = 'install C/C++ header files'
-
- user_options = [('install-dir=', 'd',
- 'directory to install header files to'),
- ('force', 'f',
- 'force installation (overwrite existing files)'),
- ]
-
- boolean_options = ['force']
-
- def initialize_options(self):
- self.install_dir = None
- self.force = 0
- self.outfiles = []
-
- def finalize_options(self):
- self.set_undefined_options('install',
- ('install_headers', 'install_dir'),
- ('force', 'force'))
-
- def mkdir_and_copy_file(self, header):
- if 'pb.h' in header:
- install_dir = re.sub('${PADDLE_BINARY_DIR}/', '', header)
- elif 'third_party' not in header:
- # paddle headers
- install_dir = re.sub('@PADDLE_SOURCE_DIR@/', '', header)
- print('install_dir: ', install_dir)
- if 'fluid/jit' in install_dir:
- install_dir = re.sub('fluid/jit', 'jit', install_dir)
- print('fluid/jit install_dir: ', install_dir)
- if 'trace_event.h' in install_dir:
- install_dir = re.sub('fluid/platform/profiler', 'phi/backends/custom', install_dir)
- print('trace_event.h install_dir: ', install_dir)
- else:
- # third_party
- install_dir = re.sub('${THIRD_PARTY_PATH}', 'third_party', header)
- patterns = ['install/mkldnn/include', 'pybind/src/extern_pybind/include', 'third_party/xpu/src/extern_xpu/xpu/include/']
- for pattern in patterns:
- install_dir = re.sub(pattern, '', install_dir)
- install_dir = os.path.join(self.install_dir, os.path.dirname(install_dir))
- if not os.path.exists(install_dir):
- self.mkpath(install_dir)
- return self.copy_file(header, install_dir)
-
- def run(self):
- hdrs = self.distribution.headers
- if not hdrs:
- return
- self.mkpath(self.install_dir)
- for header in hdrs:
- (out, _) = self.mkdir_and_copy_file(header)
- self.outfiles.append(out)
-
- def get_inputs(self):
- return self.distribution.headers or []
-
- def get_outputs(self):
- return self.outfiles
-
- class EggInfo(egg_info):
- """Copy license file into `.dist-info` folder."""
-
- def run(self):
- # don't duplicate license into `.dist-info` when building a distribution
- if not self.distribution.have_run.get('install', True):
- self.mkpath(self.egg_info)
- self.copy_file("@PADDLE_SOURCE_DIR@/LICENSE", self.egg_info)
-
- egg_info.run(self)
-
- # we redirect setuptools log for non-windows
- if sys.platform != 'win32':
- @contextmanager
- def redirect_stdout():
- f_log = open('${SETUP_LOG_FILE}', 'w')
- origin_stdout = sys.stdout
- sys.stdout = f_log
- yield
- f_log = sys.stdout
- sys.stdout = origin_stdout
- f_log.close()
- else:
- @contextmanager
- def redirect_stdout():
- yield
-
- # Log for PYPI
- with open("@PADDLE_BINARY_DIR@/python/paddle/README.md", "r", encoding='UTF-8') as f:
- long_description = f.read()
-
- # strip *.so to reduce package size
- if '${WITH_STRIP}' == 'ON':
- command = 'find ${PADDLE_BINARY_DIR}/python/paddle -name "*.so" | xargs -i strip {}'
- if os.system(command) != 0:
- raise Exception("strip *.so failed, command: %s" % command)
-
- with redirect_stdout():
- setup(name='${PACKAGE_NAME}',
- version='${PADDLE_VERSION}',
- description='Parallel Distributed Deep Learning',
- long_description=long_description,
- long_description_content_type="text/markdown",
- author_email="Paddle-better@baidu.com",
- maintainer="PaddlePaddle",
- maintainer_email="Paddle-better@baidu.com",
- project_urls = {
- 'Homepage': 'https://www.paddlepaddle.org.cn/',
- 'Downloads': 'https://github.com/paddlepaddle/paddle'
- },
- license='Apache Software License',
- packages=packages,
- install_requires=setup_requires,
- ext_modules=ext_modules,
- package_data=package_data,
- package_dir=package_dir,
- scripts=paddle_bins,
- distclass=BinaryDistribution,
- headers=headers,
- cmdclass={
- 'install_headers': InstallHeaders,
- 'install': InstallCommand,
- 'egg_info': EggInfo,
- },
- entry_points={
- 'console_scripts': [
- 'fleetrun = paddle.distributed.launch.main:launch'
- ]
- },
- classifiers=[
- 'Development Status :: 5 - Production/Stable',
- 'Operating System :: OS Independent',
- 'Intended Audience :: Developers',
- 'Intended Audience :: Education',
- 'Intended Audience :: Science/Research',
- 'License :: OSI Approved :: Apache Software License',
- 'Programming Language :: C++',
- 'Programming Language :: Python :: 3.7',
- 'Programming Language :: Python :: 3.8',
- 'Programming Language :: Python :: 3.9',
- 'Programming Language :: Python :: 3.10',
- ],
- )
-
- # As there are a lot of files in purelib which causes many logs,
- # we don't print them on the screen, and you can open `setup.py.log`
- # for the full logs.
- if os.path.exists('${SETUP_LOG_FILE}'):
- os.system('grep -v "purelib" ${SETUP_LOG_FILE}')
|