From 2ebda8e88aa113a4c46255e5e0804af50bb85cbb Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Tue, 20 Oct 2026 13:43:57 +0800 Subject: [PATCH] [None][chore] Remove legacy alternativeTRT CI paths Signed-off-by: Yanchao Lu --- jenkins/Build.groovy | 16 +----- jenkins/L0_MergeRequest.groovy | 56 +++++++------------ jenkins/L0_Test.groovy | 99 +++++++++------------------------- jenkins/scripts/cbts/README.md | 2 +- 4 files changed, 47 insertions(+), 126 deletions(-) diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index 6ade8144e487..6cbbddc5280a 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -693,9 +693,6 @@ def runLLMBuild( sh "rm -rf **/*.xml *.tar.gz" trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) - if (env.alternativeTRT) { - sh "cd ${LLM_ROOT} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" - } sh "mkdir TensorRT-LLM" sh "cp -r ${LLM_ROOT}/ TensorRT-LLM/src/" @@ -722,10 +719,6 @@ def runLLMBuild( // install python package trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${LLM_ROOT} && pip3 install -r requirements-dev.txt ${pipArgs}") - if (env.alternativeTRT) { - trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, "cp312") - } - int buildJobs = (buildFlags[BUILD_JOBS_FOR_CONFIG] ?: BUILD_JOBS) as int withEnv([ @@ -949,7 +942,7 @@ def applyLatestBolt(pipeline, tarName, is_linux_x86_64, artifacts=null) } } -def buildWheelInContainer(pipeline, libraries=[], triple=X86_64_TRIPLE, clean=false, pre_cxx11abi=false, cpver="312", extra_args="", reconfigure=false) +def buildWheelInContainer(pipeline, libraries=[], triple=X86_64_TRIPLE, clean=false, pre_cxx11abi=false, extra_args="", reconfigure=false) { // Random sleep to avoid resource contention sleep(10 * Math.random()) @@ -960,10 +953,6 @@ def buildWheelInContainer(pipeline, libraries=[], triple=X86_64_TRIPLE, clean=fa // Step 1: cloning tekit source code trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) - if (env.alternativeTRT) { - trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, cpver) - sh "cd ${LLM_ROOT} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" - } // Step 2: building libs in container sh "bash -c 'pip3 show tensorrt || true'" @@ -1003,7 +992,6 @@ def launchStages(pipeline, cpu_arch, enableFailFast, globalVars) echo "dockerImage: ${env.dockerImage}" echo "gitlabSourceRepoHttpUrl: ${env.gitlabSourceRepoHttpUrl}" echo "gitlabCommit: ${env.gitlabCommit}" - echo "alternativeTRT: ${env.alternativeTRT}" echo "Using GitLab repo: ${LLM_REPO}. Commit: ${env.gitlabCommit}" echo "env.globalVars is: ${env.globalVars}" @@ -1124,7 +1112,7 @@ def launchStages(pipeline, cpu_arch, enableFailFast, globalVars) stage(key) { stage("[${key}] Run") { echoNodeAndGpuInfo(pipeline, key) - buildWheelInContainer(pipeline, [], X86_64_TRIPLE, false, false, "cp312", "-a '90' -b Debug --micro_benchmarks", true) + buildWheelInContainer(pipeline, [], X86_64_TRIPLE, false, false, "-a '90' -b Debug --micro_benchmarks", true) } } }) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index df7517c56e14..398466596268 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -828,9 +828,7 @@ def launchReleaseCheck(pipeline, globalVars) // Step 2: Run guardwords scan def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) - if (env.alternativeTRT || - isOfficialPostMergeJob || - runMode == "nightly_release") { + if (isOfficialPostMergeJob || runMode == "nightly_release") { trtllm_utils.checkoutSource(SCAN_REPO, SCAN_COMMIT, SCAN_ROOT, false, true) trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${SCAN_ROOT} && pip3 install -e .") try { @@ -866,7 +864,7 @@ def launchReleaseCheck(pipeline, globalVars) // Step 3: Run pre-commit checks // Post-merge CI runs on all files; pre-merge CI runs only on changed files. def precommitArgs = "-a" - if (!(env.JOB_NAME ==~ /.*PostMerge.*/ || env.alternativeTRT)) { + if (!isOfficialPostMergeJob) { // Use GitLab/GitHub API to get the exact list of changed files in this MR. // This avoids git history depth issues with shallow clones. def changedFileList = getMergeRequestChangedFileList(pipeline, globalVars) @@ -1223,7 +1221,6 @@ def requireMultiGpuApprovalLabel(pipeline, globalVars, String arch) { def getMergeRequestChangedFileList(pipeline, globalVars) { def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) if ((params.InfraDryRun?.toString()?.toBoolean() ?: false) || - env.alternativeTRT || isOfficialPostMergeJob || runMode == "nightly_release") { pipeline.echo("Force set changed file list to empty list.") @@ -1253,7 +1250,6 @@ def getMergeRequestOneFileChanges(pipeline, globalVars, filePath) { // If there is an error to get the changed file diff, skip merging the waive list. def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) if ((params.InfraDryRun?.toString()?.toBoolean() ?: false) || - env.alternativeTRT || isOfficialPostMergeJob || runMode == "nightly_release") { pipeline.echo("Force set changed file diff to empty string.") @@ -1268,7 +1264,7 @@ def getMergeRequestOneFileChanges(pipeline, globalVars, filePath) { def getAutoTriggerTagList(pipeline, testFilter, globalVars) { def autoTriggerTagList = [] def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) - if (env.alternativeTRT || isOfficialPostMergeJob) { + if (isOfficialPostMergeJob) { pipeline.echo("Force set auto trigger tags to empty list.") return autoTriggerTagList } @@ -1342,8 +1338,8 @@ def getCbtsResult(pipeline, testFilter, globalVars) } def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) - if (env.alternativeTRT || isOfficialPostMergeJob) { - pipeline.echo("CBTS: deferring — post-merge job or alternativeTRT set") + if (isOfficialPostMergeJob) { + pipeline.echo("CBTS: deferring — post-merge job") return null } @@ -1844,7 +1840,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) pipeline.echo("Force not run multi-GPU testing.") return false } - if (env.alternativeTRT || testFilter[(ADD_MULTI_GPU_TEST)] || testFilter[(ONLY_MULTI_GPU_TEST)] || testFilter[(IS_POST_MERGE)]) { + if (testFilter[(ADD_MULTI_GPU_TEST)] || testFilter[(ONLY_MULTI_GPU_TEST)] || testFilter[(IS_POST_MERGE)]) { pipeline.echo("Force run multi-GPU testing.") return true } @@ -2016,7 +2012,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) def getOnlyOneGroupChanged(pipeline, testFilter, globalVars) { def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) - if (env.alternativeTRT || isOfficialPostMergeJob) { + if (isOfficialPostMergeJob) { pipeline.echo("Force set ONLY_ONE_GROUP_CHANGED \"\".") return "" } @@ -2469,7 +2465,7 @@ def findCachedPLCSourceScanResult(commit) { return [result: buildInfo.result, url: buildInfo.url] } -def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platform="x86_64", additionalParameters = [:]) { +def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, additionalParameters = [:]) { def parameters = getCommonParameters() // Build a local copy to avoid racey growth from shared parallel mutations. // In particular, CACHED_CHANGED_FILE_LIST can become very large and may @@ -2482,18 +2478,6 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor 'globalVars': globalVarsJson, ] + additionalParameters - if (env.alternativeTRT && platform == "x86_64") { - parameters += [ - 'alternativeTRT': env.alternativeTRT, - ] - } - - if (env.alternativeTrtSBSA && platform == "SBSA") { - parameters += [ - 'alternativeTRT': env.alternativeTrtSBSA, - ] - } - // Preserve an explicit empty phase override from the dry-run helper. if (!additionalParameters.containsKey('testPhase2StageName') && env.testPhase2StageName) { parameters += [ @@ -2540,7 +2524,7 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor return buildStatus } -def launchInfraDryRunTestJob(pipeline, arch, testFilter, globalVars, platform, imageParameters) +def launchInfraDryRunTestJob(pipeline, arch, testFilter, globalVars, imageParameters) { String testFilterJson = writeJSON returnText: true, json: testFilter def additionalParameters = [ @@ -2549,7 +2533,7 @@ def launchInfraDryRunTestJob(pipeline, arch, testFilter, globalVars, platform, i 'testPhase2StageName': '', ] + imageParameters stage("[Test-${arch}-Single-GPU] Remote Run") { - launchJob(pipeline, "L0_Test-${arch}-Single-GPU", false, false, globalVars, platform, additionalParameters) + launchJob(pipeline, "L0_Test-${arch}-Single-GPU", false, false, globalVars, additionalParameters) } } @@ -2694,7 +2678,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) // launchJob returns UNSTABLE (without throwing) when the build // sub-job was infra-incomplete: only infra aborts, no genuine // build failure (see runBranchesWithInfraDefer in Build.groovy). - def buildStatus = launchJob(pipeline, "/LLM/helpers/Build-x86_64", reuseBuild, enableFailFast, globalVars, "x86_64", additionalParameters) + def buildStatus = launchJob(pipeline, "/LLM/helpers/Build-x86_64", reuseBuild, enableFailFast, globalVars, additionalParameters) buildInfraIncomplete = (buildStatus == "UNSTABLE") } @@ -2712,7 +2696,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) 'wheelDockerImagePy310': globalVars["LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE"], 'wheelDockerImagePy312': globalVars["LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE"], ] - launchInfraDryRunTestJob(pipeline, "x86_64", testFilter, globalVars, "x86_64", imageParameters) + launchInfraDryRunTestJob(pipeline, "x86_64", testFilter, globalVars, imageParameters) return } @@ -2750,7 +2734,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) // launchJob returns UNSTABLE (without throwing) when the single-GPU // sub-job was infra-incomplete: only infra aborts, no real failure. - def singleGpuStatus = launchJob(pipeline, "L0_Test-x86_64-Single-GPU", false, enableFailFast, globalVars, "x86_64", additionalParameters) + def singleGpuStatus = launchJob(pipeline, "L0_Test-x86_64-Single-GPU", false, enableFailFast, globalVars, additionalParameters) singleGpuInfraIncomplete = (singleGpuStatus == "UNSTABLE") } catch (InterruptedException e) { throw e @@ -2839,7 +2823,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) 'wheelDockerImagePy312': globalVars["LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE"], ] - launchJob(pipeline, "L0_Test-x86_64-Multi-GPU", false, enableFailFast, globalVars, "x86_64", additionalParameters) + launchJob(pipeline, "L0_Test-x86_64-Multi-GPU", false, enableFailFast, globalVars, additionalParameters) } catch (InterruptedException e) { throw e @@ -2880,7 +2864,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) // launchJob returns UNSTABLE (without throwing) when the build // sub-job was infra-incomplete: only infra aborts, no genuine // build failure (see runBranchesWithInfraDefer in Build.groovy). - def buildStatus = launchJob(pipeline, "/LLM/helpers/Build-SBSA", reuseBuild, enableFailFast, globalVars, "SBSA", additionalParameters) + def buildStatus = launchJob(pipeline, "/LLM/helpers/Build-SBSA", reuseBuild, enableFailFast, globalVars, additionalParameters) buildInfraIncomplete = (buildStatus == "UNSTABLE") } @@ -2898,7 +2882,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) "dockerImage": globalVars["LLM_SBSA_DOCKER_IMAGE"], 'wheelDockerImage': globalVars["LLM_SBSA_WHEEL_DOCKER_IMAGE"], ] - launchInfraDryRunTestJob(pipeline, "SBSA", testFilter, globalVars, "SBSA", imageParameters) + launchInfraDryRunTestJob(pipeline, "SBSA", testFilter, globalVars, imageParameters) return } @@ -2933,7 +2917,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) // both writers on the same object. 'boltPublishCanonical': "false", ] - launchJob(pipeline, "/LLM/helpers/BoltProfileGen", false, false, globalVars, "SBSA", additionalParameters) + launchJob(pipeline, "/LLM/helpers/BoltProfileGen", false, false, globalVars, additionalParameters) } } } @@ -2971,7 +2955,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) // launchJob returns UNSTABLE (without throwing) when the single-GPU // sub-job was infra-incomplete: only infra aborts, no real failure. - def singleGpuStatus = launchJob(pipeline, "L0_Test-SBSA-Single-GPU", false, enableFailFast, globalVars, "SBSA", additionalParameters) + def singleGpuStatus = launchJob(pipeline, "L0_Test-SBSA-Single-GPU", false, enableFailFast, globalVars, additionalParameters) singleGpuInfraIncomplete = (singleGpuStatus == "UNSTABLE") } catch (InterruptedException e) { throw e @@ -3057,7 +3041,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) 'wheelDockerImage': globalVars["LLM_SBSA_WHEEL_DOCKER_IMAGE"], ] - launchJob(pipeline, "L0_Test-SBSA-Multi-GPU", false, enableFailFast, globalVars, "SBSA", additionalParameters) + launchJob(pipeline, "L0_Test-SBSA-Multi-GPU", false, enableFailFast, globalVars, additionalParameters) } catch (InterruptedException e) { throw e @@ -3149,7 +3133,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) if (params.release_type_id) { additionalParameters['release_type_id'] = params.release_type_id } - imageBuildStatus = launchJob(pipeline, "/LLM/helpers/BuildDockerImages", false, enableFailFast, globalVars, "x86_64", additionalParameters) + imageBuildStatus = launchJob(pipeline, "/LLM/helpers/BuildDockerImages", false, enableFailFast, globalVars, additionalParameters) } catch (InterruptedException e) { throw e } catch (Exception e) { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 9c41b6cf65a6..80b39efa180a 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1178,7 +1178,7 @@ boolean isNonTerminalSlurmState(String state) { return state != null && SLURM_NON_TERMINAL_STATES.contains(state.toUpperCase(java.util.Locale.ROOT)) } -def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, skipInstallWheel=false, cpver="cp312", String postTag="", boolean useClusterDurations=false, Map placementContext=null, Map retryContext=null) +def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, skipInstallWheel=false, String postTag="", boolean useClusterDurations=false, Map placementContext=null, Map retryContext=null) { SlurmPartition partition = SlurmConfig.resolvePlatform(platform) SlurmCluster cluster = SlurmConfig.clusterConfig[partition.clusterName] @@ -1732,7 +1732,7 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, return err } try { - executeLLMTestOnSlurm(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, slurmRunner, postTag, useClusterDurations, retryContext, classifySlurmFailure) + executeLLMTestOnSlurm(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, slurmRunner, postTag, useClusterDurations, retryContext, classifySlurmFailure) } catch (InterruptedException e) { throw e } catch (Exception e) { @@ -1781,13 +1781,13 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, } } -def executeLLMTestOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", runner, String postTag="", boolean useClusterDurations=false, Map retryContext=null, Closure classifySlurmFailure=null) +def executeLLMTestOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, runner, String postTag="", boolean useClusterDurations=false, Map retryContext=null, Closure classifySlurmFailure=null) { runner { // TODO: refactor the finallyRunner to reuse within slurm or nonslurm job. cacheErrorAndUploadResult(pipeline, stageName, { try { - runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, postTag, useClusterDurations, retryContext) + runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, postTag, useClusterDurations, retryContext) } catch (InterruptedException e) { throw e } catch (Exception e) { @@ -2143,7 +2143,7 @@ def boltedTarArtifactPath(pipeline, String config, String tarName) return "${ARTIFACT_PATH}/bolted-${tarName}" } -def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, skipInstallWheel=false, cpver="cp312", String postTag="", boolean useClusterDurations=false, Map placementContext=null, Map retryContext=null) +def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, skipInstallWheel=false, String postTag="", boolean useClusterDurations=false, Map placementContext=null, Map retryContext=null) { SlurmPartition partition = SlurmConfig.resolvePlatform(platform) SlurmCluster cluster = SlurmConfig.clusterConfig[partition.clusterName] @@ -3445,7 +3445,7 @@ boolean isDispatcherPodFailure(Throwable e) { return ContextDeath.isContextDeath(e) } -def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, runWithSbatch=false, skipInstallWheel=false, cpver="cp312", String outerAttemptTag="", boolean useClusterDurations=false, Integer infraRetryMax=null) +def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, runWithSbatch=false, skipInstallWheel=false, String outerAttemptTag="", boolean useClusterDurations=false, Integer infraRetryMax=null) { echo "Run Slurm job with native sbatch: $runWithSbatch" @@ -3503,9 +3503,9 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p def postTag = "${outerAttemptTag}${innerSuffix}" if (nodeCount > 1 || runWithSbatch) { - runLLMTestlistWithSbatch(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, nodeCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext) + runLLMTestlistWithSbatch(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, nodeCount, skipInstallWheel, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext) } else { - runLLMTestlistWithAgent(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, skipInstallWheel, cpver, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext) + runLLMTestlistWithAgent(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, gpuCount, skipInstallWheel, postTag, useClusterDurations, attemptPlacementContext, slurmRetryContext) } // Job succeeded @@ -4685,9 +4685,6 @@ def runLLMDocBuild(pipeline, config) fetchBuildArtifact(pipeline, "${ARTIFACT_PATH}/${tarName}", "${llmPath}/${tarName}") sh "cd ${llmPath} && tar -zxf ${tarName}" // install python package - if (env.alternativeTRT) { - sh "cd ${llmSrc} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" - } trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmSrc} && pip3 install -r requirements-dev.txt") trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && pip3 install --force-reinstall --no-deps TensorRT-LLM/tensorrt_llm-*.whl") @@ -5683,7 +5680,7 @@ def priorAttemptTags(String postTag) { return priors } -def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", String postTag="", boolean useClusterDurations=false, Map retryContext=null) +def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, String postTag="", boolean useClusterDurations=false, Map retryContext=null) { // Step 1: create LLM_ROOT dir and clean up the workspace def llmRootConfig = "${LLM_ROOT}${config}" @@ -5700,12 +5697,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def llmSrc = "${llmPath}/TensorRT-LLM/src" echoNodeAndGpuInfo(pipeline, stageName) - if (env.alternativeTRT && cpver) { - stage("Replace TensorRT") { - trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, cpver) - } - } - // Step 2: run tests stage ("Setup Environment") { @@ -5760,9 +5751,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO // pre-installs a platform-specific wheel) — install deps but skip the wheel. timeout(time: 45, unit: 'MINUTES') { if (!isFatSqsh) { - if (env.alternativeTRT) { - sh "cd ${llmSrc} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" - } trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmSrc} && pip3 install -r requirements-dev.txt") // SMG's generated package remains an opt-in dependency. OpenEngine's // private bindings and grpcio dependency are part of the base package. @@ -6210,7 +6198,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO // composed with an attempt tag by the helper) and `isFinalAttempt` (so this // function's `cacheErrorAndUploadResult` can suppress synthetic stage-fail XML // and junit() for intermediate retryable failures). -def runLLMTestlistOnPlatform(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", postTag="", boolean isFinalAttempt=true, Map retryContext=null, boolean useClusterDurations=false) +def runLLMTestlistOnPlatform(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, postTag="", boolean isFinalAttempt=true, Map retryContext=null, boolean useClusterDurations=false) { cacheErrorAndUploadResult(pipeline, stageName, { // PMIx builds a singleton ID, singleton.{hostname}.{pid}, that must fit a @@ -6225,7 +6213,7 @@ def runLLMTestlistOnPlatform(pipeline, platform, testList, config=VANILLA_CONFIG "PMIX_HOSTNAME=mpi-node0", ] withEnv(testEnv) { - runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, postTag, useClusterDurations, retryContext) + runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, postTag, useClusterDurations, retryContext) } }, { if (testFilter[(DEBUG_MODE)]) { @@ -6464,9 +6452,6 @@ def runLLMBuild( sh "ccache -sv" trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, "tensorrt_llm", true, true) - if (env.alternativeTRT) { - sh "cd tensorrt_llm/ && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" - } // Random sleep to avoid resource contention sleep(10 * Math.random()) sh "curl ifconfig.me || true" @@ -6480,9 +6465,6 @@ def runLLMBuild( trtllm_utils.llmExecStepWithRetry(pipeline, script: "#!/bin/bash \n" + "cd tensorrt_llm/ && pip3 install -r requirements-dev.txt") trtllm_utils.llmExecStepWithRetry(pipeline, script: "#!/bin/bash \n" + "cd tensorrt_llm/ && pip3 install -r requirements-grpc-smg.txt") - if (env.alternativeTRT) { - trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, cpver) - } buildArgs = "--clean --nixl_root /opt/nvidia/nvda_nixl" if (cpu_arch == AARCH64_TRIPLE) { buildArgs += " -a '90;100;103;107;120;121'" @@ -6497,9 +6479,6 @@ def runLLMBuild( ]) { trtllm_utils.llmExecStepWithRetry(pipeline, script: "#!/bin/bash \n" + "cd tensorrt_llm/ && python3 scripts/build_wheel.py --version-override \"\${TRTLLM_VERSION_OVERRIDE}\" --use_ccache -G Ninja -j ${BUILD_JOBS} -D 'WARNING_IS_ERROR=ON' --extra-cmake-vars ENABLE_BOLT_COMPATIBLE=ON ${buildArgs}${platNameArg}") } - if (env.alternativeTRT) { - sh "bash -c 'pip3 show tensorrt || true'" - } def wheelName = sh(returnStdout: true, script: 'cd tensorrt_llm/build && ls -1 *.whl').trim() @@ -6578,20 +6557,12 @@ def runLLMBuild( // Test installation in the new environment // Reserve CUDA 13.0 torch and torchvision packages def pip_keep = "^pip==|^torch==|^torchvision==" - def remove_trt = "rm -rf /usr/local/tensorrt" - if (env.alternativeTRT) { - pip_keep += "|^tensorrt==" - remove_trt = "echo keep /usr/local/tensorrt" - } sh "bash -c 'pip3 list --format=freeze | grep -Ev \"${pip_keep}\" | xargs -r pip3 uninstall -y'" - sh "bash -c 'yum remove -y libcudnn* libnccl* libcublas* && ${remove_trt}'" + sh "bash -c 'yum remove -y libcudnn* libnccl* libcublas* && rm -rf /usr/local/tensorrt'" } // Test preview installation trtllm_utils.llmExecStepWithRetry(pipeline, script: "#!/bin/bash \n" + "cd tensorrt_llm/ && pip3 install pytest build/tensorrt_llm-*.whl") - if (env.alternativeTRT) { - sh "bash -c 'pip3 show tensorrt || true'" - } // Upload attribution files when they exist def attrDir = "tensorrt_llm/cpp/build/attribution" @@ -6607,7 +6578,7 @@ def runLLMBuild( } -def runPackageSanityCheck(pipeline, wheel_path, reinstall_dependencies=false, cpver="cp312") +def runPackageSanityCheck(pipeline, wheel_path, reinstall_dependencies=false) { def whlUrl = "https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/${wheel_path}" @@ -6622,31 +6593,18 @@ def runPackageSanityCheck(pipeline, wheel_path, reinstall_dependencies=false, cp sh "pwd && ls -alh" trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget -nv ${whlUrl}") - if (env.alternativeTRT) { - trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, cpver) - sh "bash -c 'pip3 show tensorrt || true'" - } - if (reinstall_dependencies) { // Test installation in the new environment // Reserve CUDA 13.0 torch and torchvision packages def pip_keep = "^pip==|^torch==|^torchvision==" - def remove_trt = "rm -rf /usr/local/tensorrt" - if (env.alternativeTRT) { - pip_keep += "|^tensorrt==" - remove_trt = "echo keep /usr/local/tensorrt" - } sh "bash -c 'pip3 list --format=freeze | grep -Ev \"${pip_keep}\" | xargs -r pip3 uninstall -y'" - sh "bash -c 'yum remove -y libcudnn* libnccl* libcublas* && ${remove_trt}'" + sh "bash -c 'yum remove -y libcudnn* libnccl* libcublas* && rm -rf /usr/local/tensorrt'" } //WAR: remove python3-pygments first since it is installed in NGC PyTorch image trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get remove -y python3-pygments") // Test preview installation trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'pip3 install pytest tensorrt_llm-*.whl'") - if (env.alternativeTRT) { - sh "bash -c 'pip3 show tensorrt || true'" - } def pkgUrl = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/${linuxPkgName}" trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget -nv ${pkgUrl}") @@ -7104,7 +7062,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) if (key.contains("llvm")) { config = LLVM_CONFIG } - runLLMTestlistOnPlatform(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], false, "cp312", attemptTag, isFinalAttempt, retryContext) + runLLMTestlistOnPlatform(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], false, attemptTag, isFinalAttempt, retryContext) }]]} fullSet = parallelJobs.keySet() @@ -7186,7 +7144,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) if (key.contains("llvm")) { config = LLVM_CONFIG } - runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag, false, retryContext?.infraRetryMax) + runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, attemptTag, false, retryContext?.infraRetryMax) }, [singleAttempt: true, slurmDispatcher: true]]]} // SLURM dispatcher pods run their own inner retry loop // (runLLMTestlistOnSlurm with SLURM_INFRA_RETRY_MAX). Disabling the outer @@ -7399,7 +7357,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) if (env.targetArch == AARCH64_TRIPLE) { parallelJobs = SBSATestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, values[0], "arm64"), { attemptTag, isFinalAttempt, retryContext = null -> - runLLMTestlistOnPlatform(pipeline, values[0], values[1], LINUX_AARCH64_CONFIG, false, key, values[2], values[3], false, "cp312", attemptTag, isFinalAttempt, retryContext, values[4] ?: false) + runLLMTestlistOnPlatform(pipeline, values[0], values[1], LINUX_AARCH64_CONFIG, false, key, values[2], values[3], false, attemptTag, isFinalAttempt, retryContext, values[4] ?: false) }]]} // Add SBSA Slurm jobs @@ -7418,7 +7376,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) if (key.contains("llvm")) { config = LLVM_CONFIG } - runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false, retryContext?.infraRetryMax) + runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, attemptTag, values[7] ?: false, retryContext?.infraRetryMax) }, [singleAttempt: true, slurmDispatcher: true]]]} parallelJobs += parallelSlurmJobs @@ -7432,7 +7390,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) if (key.contains("llvm")) { config = LLVM_CONFIG } - runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 2, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false, retryContext?.infraRetryMax) + runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 2, values[6] ?: false, false, attemptTag, values[7] ?: false, retryContext?.infraRetryMax) }, [singleAttempt: true, slurmDispatcher: true]]]} parallelJobs += parallelMultiNodesSBSAJobs @@ -7591,10 +7549,8 @@ def launchTestJobs(pipeline, testFilter, globalVars) def isDlfw = values[4] def wheelPath = "" def cpver = "cp312" - def pyver = "3.12" if (key.contains("PY310")) { cpver = "cp310" - pyver = "3.10" } // Same switch the build helpers use for the tarball, so the released @@ -7615,7 +7571,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) // TODO: Re-enable the sanity check after updating GPU testers' driver version. // sanityRunner("Sanity check") { - // runPackageSanityCheck(pipeline, wheelPath, values[3], cpver) + // runPackageSanityCheck(pipeline, wheelPath, values[3]) // } def checkPipStage = false @@ -7682,13 +7638,6 @@ def launchTestJobs(pipeline, testFilter, globalVars) "PRTE_ALLOW_RUN_AS_ROOT_CONFIRM=1", "PMIX_HOSTNAME=mpi-node0", ] - if (env.alternativeTRT) { - stage("Replace TensorRT") { - trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, cpver) - } - libEnv += ["LD_LIBRARY_PATH+tensorrt=/usr/local/tensorrt/lib"] - libEnv += ["LD_LIBRARY_PATH+nvrtc=/usr/local/lib/python${pyver}/dist-packages/nvidia/cuda_nvrtc/lib"] - } echo "###### Check pip install Start ######" withEnv(libEnv) { // Retry 2 times if timeout occurs. @@ -7721,7 +7670,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) } withEnv(libEnv) { sh "env | sort" - runLLMTestlistOnPlatform(pipeline, gpu_type, "l0_sanity_check", config, false, toStageName(values[1], key), 1, 1, true, cpver, "-SubJob-RunTest" + attemptTag, isFinalAttempt, retryContext) + runLLMTestlistOnPlatform(pipeline, gpu_type, "l0_sanity_check", config, false, toStageName(values[1], key), 1, 1, true, "-SubJob-RunTest" + attemptTag, isFinalAttempt, retryContext) } }) } @@ -7762,9 +7711,9 @@ def launchTestJobs(pipeline, testFilter, globalVars) println parallelJobsFiltered.keySet() } - // Check --post-merge, post-merge or TRT dependency testing pipelines. + // Check --post-merge or post-merge pipelines. // If true, add post-merge only test stages and multi-GPU test stages. - if (env.alternativeTRT || testFilter[(IS_POST_MERGE)]) { + if (testFilter[(IS_POST_MERGE)]) { parallelJobsFiltered += multiGpuJobs parallelJobsFiltered += postMergeJobs } @@ -8007,7 +7956,7 @@ def launchTestJobsForImagesSanityCheck(pipeline, globalVars) { runKubernetesPodWithInfraRetry(pipeline, imageSanitySpec, "trt-llm", values.name, { attemptTag, isFinalAttempt, retryContext = null -> sh "env | sort" trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y git rsync curl") - runLLMTestlistOnPlatform(pipeline, values.gpuType, "l0_sanity_check", values.config, false, values.name, 1, 1, true, null, "-SubJob-TestImage" + attemptTag, isFinalAttempt, retryContext) + runLLMTestlistOnPlatform(pipeline, values.gpuType, "l0_sanity_check", values.config, false, values.name, 1, 1, true, "-SubJob-TestImage" + attemptTag, isFinalAttempt, retryContext) }) } } else { diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 20453820c954..324958863b62 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -358,7 +358,7 @@ combined via `_combine_scopes`. CBTS defers to the existing filter chain when: -- PostMerge job / `alternativeTRT` set +- PostMerge job - `changed_files` is empty - `cbts.command main` throws or stdout is unparsable - Python returns `scope: null`