Explorar el Código

Jenkins 从UAT生成生产环境镜像脚本

dujian hace 3 días
padre
commit
e36b57aee6
Se han modificado 1 ficheros con 190 adiciones y 54 borrados
  1. 190 54
      docs/jenkins/Jenkinsfile-prod-promote-from-uat.groovy

+ 190 - 54
docs/jenkins/Jenkinsfile-prod-promote-from-uat.groovy

@@ -45,7 +45,8 @@
  *   <PROD_DEPLOY_ROOT>/<prodDir>/config/bootstrap-prod.yml(若同步 bootstrap)
  * 与预生产「每服务一子目录」约定一致。生产 Docker 挂载 jar 时请指向 PROD 路径。
  * 若 compose 将生产 jar 挂为「../java:/app_deploy」(相对 compose 目录),则 Jenkins 内应设 PROD_DEPLOY_ROOT=/app_deploy,对应宿主机为 compose 上级目录下的 java/。
- * 【必配】若 Job 使用 PROD_DEPLOY_ROOT=/alien_produ/java,Jenkins 容器必须 volume 挂载宿主机 /alien_produ/java 到同路径(见 docker-compose-jenkins-host-network.yml),否则 cp 写在 Jenkins 私有层、生产侧进程读宿主机文件时,unzip 通过仍 corrupt jarfile。
+ * 【必配】若 Job 使用 PROD_DEPLOY_ROOT=/alien_produ/java 且 **未** 配置 PROD_SSH_TARGET(本机发版):Jenkins 容器必须 volume 挂载宿主机 /alien_produ/java 到同路径(见 docker-compose-jenkins-host-network.yml),否则 cp 写在 Jenkins 私有层、生产侧进程读宿主机文件时,unzip 通过仍 corrupt jarfile。
+ * 【环境隔离】预生产仍在 39.106.135.88(UAT_DEPLOY_ROOT 读该机上制品);生产默认发到 **39.105.153.68**:设置 PROD_SSH_TARGET=alien_store@39.105.153.68(可改)后,制品先写入 Jenkins 工作区下暂存目录再 rsync 到目标机 PROD_DEPLOY_ROOT,docker/docker compose 经 SSH 在目标机执行(须 Jenkins 到目标机免密 sudo docker)。留空 PROD_SSH_TARGET 则保持「Jenkins 与生产 dockerd 同机」旧行为。
  *
  * 生产容器须由 <PROD_DEPLOY_ROOT>/docker-compose.yml(或 PROD_COMPOSE_FILE)通过 docker compose 管理;与 docker restart / docker run 的「独立容器」不共享 Compose 元数据,见 Job 头「docker compose 启动方式」说明。
  * 若 PROD 目录下出现 alien-xxx-1.0.0.jar「目录」且为空:多为历史上 docker -v 指向不存在的文件时 Docker 在宿主机误建同名目录;须 rm -rf 该目录后再拷贝真 jar。
@@ -297,7 +298,13 @@ pipeline {
                 name: 'PROD_DEPLOY_ROOT',
                 defaultValue: '/alien_produ/java',
                 trim: true,
-                description: 'Jenkins 进程内可见的生产制品根目录(无末尾斜杠)。现网 compose 若挂 ../java:/app_deploy,则填 /app_deploy;若生产目录在宿主机 /alien_produ/java 且已挂载到容器同路径,则填 /alien_produ/java'
+                description: '生产制品根目录(无末尾斜杠),须与目标机 compose 卷一致。本机发版时为 Jenkins 可写路径;配置 PROD_SSH_TARGET 时为「目标机」上的绝对路径(如 /alien_produ/java),制品经 rsync 写入该路径'
+        )
+        string(
+                name: 'PROD_SSH_TARGET',
+                defaultValue: 'alien_store@39.105.153.68',
+                trim: true,
+                description: '生产 Docker 所在主机 SSH 目标(如 alien_store@39.105.153.68)。留空=旧行为:在本机 PROD_DEPLOY_ROOT 写文件并用本机 docker compose。非空时:先写入工作区 .jenkins_remote_prod_staging 再 rsync 到对方 PROD_DEPLOY_ROOT,且 docker 命令经「ssh 目标 sudo docker」执行;须配置免密登录与目标机 sudo 免密 docker'
         )
         string(
                 name: 'PROD_PAY_CERT_HOST',
@@ -359,6 +366,13 @@ pipeline {
                     echo ">>> bootstrap 同步模式: ${params.BOOTSTRAP_SYNC_MODE}"
                     echo ">>> UAT_DEPLOY_ROOT=${env.UAT_DEPLOY_ROOT}(Jenkins 进程内路径;宿主机 UAT 常为 /alien_uat/java 映射至容器 /app_deploy_uat)"
                     echo ">>> PROD_DEPLOY_ROOT=${env.PROD_DEPLOY_ROOT}"
+                    def _ssh = (params.PROD_SSH_TARGET ?: '').trim()
+                    if (_ssh) {
+                        echo ">>> PROD_SSH_TARGET=${_ssh}(远程生产:制品 rsync 到对方 PROD_DEPLOY_ROOT,docker compose 在目标机执行)"
+                        echo ">>> 本地暂存目录(写入后同步): ${env.WORKSPACE}/.jenkins_remote_prod_staging"
+                    } else {
+                        echo ">>> PROD_SSH_TARGET: (空)本机生产路径 + 本机 Docker"
+                    }
                     def _cf = (params.PROD_COMPOSE_FILE ?: '').trim() ?: "${env.PROD_DEPLOY_ROOT}/docker-compose.yml"
                     def _jic = (params.PROD_JAR_IN_CONTAINER ?: 'app.jar').trim() ?: 'app.jar'
                     echo ">>> PROD_COMPOSE_FILE 实际使用: ${_cf}(docker compose -f 且 --project-directory=所在目录)"
@@ -390,7 +404,7 @@ pipeline {
                         echo ">>> PROD_DOCKER_CREATE_EXTERNAL_NETWORKS: ${_enList}(缺失则 create;多网用逗号)"
                     }
                     echo ">>> PROD_JAR_IN_CONTAINER: ${_jic} => 容器内 /app/${_jic}(与 yml 卷目标一致)"
-                    echo ">>> 若 PROD_DEPLOY_ROOT 为 /alien_produ/java:Jenkins 须 volume 挂载宿主机与 compose 同路径,否则 cp 的 jar 与生产容器读到的不是同一份文件"
+                    echo ">>> 若未配置 PROD_SSH_TARGET 且 PROD_DEPLOY_ROOT 为 /alien_produ/java:Jenkins 须 volume 挂载宿主机与 compose 同路径,否则 cp 的 jar 与生产容器读到的不是同一份文件"
                     def pch = (params.PROD_PAY_CERT_HOST ?: '').trim()
                     if (pch) {
                         echo ">>> PROD_PAY_CERT_HOST=${pch}(证书卷全局覆盖:store / store-platform / lawyer / dining -> /usr/local/alien/aliPayCert)"
@@ -424,24 +438,44 @@ pipeline {
             steps {
                 script {
                     def prodImg = params.PROD_JRE_IMAGE ?: 'my-openjdk8-ffmpeg:v1'
-                    sh """
-                        set -e
-                        PROD="${env.PROD_DEPLOY_ROOT}"
-                        IMG="${prodImg}"
-                        PFILE=".jenkins_jar_promote_mount_probe"
-                        rm -f "\$PROD/\$PFILE" 2>/dev/null || true
-                        printf ok > "\$PROD/\$PFILE"
-                        OUT=\$(docker run --rm --entrypoint cat -v "\$PROD:/__prod:ro" "\$IMG" "/__prod/\$PFILE" 2>/dev/null || true)
-                        if [ "\$OUT" != "ok" ]; then
-                            echo "ERROR: Jenkins 在 \\\$PROD 下的写入,无法从「Docker 以宿主机路径 -v \\\$PROD」挂载的容器中读到。"
-                            echo "原因:Jenkins 容器内 \\\$PROD 未绑定到宿主机同路径,cp 写在容器私有文件系统,生产侧仍可能读宿主机旧/空文件。"
-                            echo "处理:在 Jenkins docker-compose 增加 volume,例如:- /alien_produ/java:/alien_produ/java(与现网生产目录一致),然后 recreate Jenkins。"
+                    def prodSshV = (params.PROD_SSH_TARGET ?: '').trim()
+                    def prodRemote = (params.PROD_DEPLOY_ROOT ?: env.PROD_DEPLOY_ROOT ?: '/alien_produ/java').trim()
+                    if (prodSshV) {
+                        sh """
+                            set -e
+                            SSH='${prodSshV.replace("'", "'\\''")}'
+                            PROD='${prodRemote.replace("'", "'\\''")}'
+                            IMG='${prodImg.replace("'", "'\\''")}'
+                            PFILE=".jenkins_jar_promote_mount_probe"
+                            ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new "\$SSH" "rm -f \"\$PROD/\$PFILE\" 2>/dev/null; printf ok | sudo tee \"\$PROD/\$PFILE\" >/dev/null"
+                            OUT=\$(ssh -o BatchMode=yes "\$SSH" "sudo docker run --rm --entrypoint cat -v \"\$PROD:/__prod:ro\" \"\$IMG\" \"/__prod/\$PFILE\" 2>/dev/null" || true)
+                            ssh -o BatchMode=yes "\$SSH" "sudo rm -f \"\$PROD/\$PFILE\" 2>/dev/null || true"
+                            if [ "\$OUT" != "ok" ]; then
+                                echo "ERROR: 远程 \\\$SSH 上路径 \\\$PROD 无法被该机的 docker 挂载读取(请核对目录权限、docker 与 ssh 用户)。"
+                                exit 1
+                            fi
+                            echo ">>> 远程 PROD 与 Docker 挂载校验通过: \$SSH:\$PROD"
+                        """
+                    } else {
+                        sh """
+                            set -e
+                            PROD="${env.PROD_DEPLOY_ROOT}"
+                            IMG="${prodImg}"
+                            PFILE=".jenkins_jar_promote_mount_probe"
                             rm -f "\$PROD/\$PFILE" 2>/dev/null || true
-                            exit 1
-                        fi
-                        rm -f "\$PROD/\$PFILE"
-                        echo ">>> PROD 与 Docker 宿主机挂载校验通过: \$PROD"
-                    """
+                            printf ok > "\$PROD/\$PFILE"
+                            OUT=\$(docker run --rm --entrypoint cat -v "\$PROD:/__prod:ro" "\$IMG" "/__prod/\$PFILE" 2>/dev/null || true)
+                            if [ "\$OUT" != "ok" ]; then
+                                echo "ERROR: Jenkins 在 \\\$PROD 下的写入,无法从「Docker 以宿主机路径 -v \\\$PROD」挂载的容器中读到。"
+                                echo "原因:Jenkins 容器内 \\\$PROD 未绑定到宿主机同路径,cp 写在容器私有文件系统,生产侧仍可能读宿主机旧/空文件。"
+                                echo "处理:在 Jenkins docker-compose 增加 volume,例如:- /alien_produ/java:/alien_produ/java(与现网生产目录一致),然后 recreate Jenkins。"
+                                rm -f "\$PROD/\$PFILE" 2>/dev/null || true
+                                exit 1
+                            fi
+                            rm -f "\$PROD/\$PFILE"
+                            echo ">>> PROD 与 Docker 宿主机挂载校验通过: \$PROD"
+                        """
+                    }
                 }
             }
         }
@@ -478,6 +512,12 @@ pipeline {
             }
             steps {
                 script {
+                    def prodSsh = (params.PROD_SSH_TARGET ?: '').trim()
+                    def prodRemoteRoot = (params.PROD_DEPLOY_ROOT ?: env.PROD_DEPLOY_ROOT ?: '/alien_produ/java').trim()
+                    def prodLocalRoot = prodSsh ? "${env.WORKSPACE}/.jenkins_remote_prod_staging".replace('\\', '/') : (env.PROD_DEPLOY_ROOT ?: '/alien_produ/java')
+                    def sshQE = prodSsh.replace("'", "'\\''")
+                    def remQE = prodRemoteRoot.replace("'", "'\\''")
+
                     def services = getServiceDefinitions()
                     def selected = filterServices(services)
                     def mode = params.BOOTSTRAP_SYNC_MODE
@@ -486,9 +526,17 @@ pipeline {
                     for (def s in selected) {
                         def jar = "${s.module}-1.0.0.jar"
                         def srcJar = "${env.UAT_DEPLOY_ROOT}/${s.uatDir}/${(jar)}"
-                        def dstDir = "${env.PROD_DEPLOY_ROOT}/${s.prodDir}/config"
+                        def dstDir = "${prodLocalRoot}/${s.prodDir}/config"
                         def dstFile = "${dstDir}/bootstrap-prod.yml"
                         def srcGit = "${WORKSPACE}/${s.bootstrapRel}"
+                        def dstDirSh = dstDir.replace("'", "'\\''")
+                        def rsyncBootstrapBlock = ''
+                        if (prodSsh) {
+                            rsyncBootstrapBlock = """
+                                    ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new '${sshQE}' "mkdir -p '${remQE}/${s.prodDir}/config'"
+                                    rsync -az '${dstDirSh}/' '${sshQE}:${remQE}/${s.prodDir}/config/'
+"""
+                        }
 
                         if (mode == 'from_jar') {
                             sh """
@@ -505,6 +553,9 @@ pipeline {
                                 fi
                                 if [ "\$JENKINS_DRY_RUN" = 'true' ]; then
                                     echo "[DRY_RUN] 尝试 BOOT-INF/classes/ 或瘦 jar 根目录 bootstrap-prod.yml -> ${dstFile}"
+                                    if [ -n '${sshQE}' ]; then
+                                        echo "[DRY_RUN] rsync config/ -> ${sshQE}:${remQE}/${s.prodDir}/config/"
+                                    fi
                                 else
                                     mkdir -p "${dstDir}"
                                     if unzip -l "${(srcJar)}" | grep -q 'BOOT-INF/classes/bootstrap-prod.yml'; then
@@ -517,6 +568,7 @@ pipeline {
                                         echo "ERROR: jar 内未找到 bootstrap-prod.yml(既无 BOOT-INF/classes/ 也无根目录,请确认已打包进 ${s.module})"
                                         exit 1
                                     fi
+                                    ${rsyncBootstrapBlock}
                                 fi
                             """
                         } else {
@@ -531,10 +583,14 @@ pipeline {
                                 if [ "\$JENKINS_DRY_RUN" = 'true' ]; then
                                     echo "[DRY_RUN] mkdir -p ${dstDir}"
                                     echo "[DRY_RUN] cp -f ${srcGit} ${dstFile}"
+                                    if [ -n '${sshQE}' ]; then
+                                        echo "[DRY_RUN] rsync config/ -> ${sshQE}:${remQE}/${s.prodDir}/config/"
+                                    fi
                                 else
                                     mkdir -p "${dstDir}"
                                     cp -f "${srcGit}" "${dstFile}"
                                     echo ">>> 已从 Git 拷贝: ${dstFile}"
+                                    ${rsyncBootstrapBlock}
                                 fi
                             """
                         }
@@ -552,6 +608,13 @@ pipeline {
                     def selected = filterServices(services)
                     def dryRunSh = dryRunForShell()
 
+                    def prodSsh = (params.PROD_SSH_TARGET ?: '').trim()
+                    def useRemoteProd = prodSsh.length() > 0
+                    def prodRemoteRoot = (params.PROD_DEPLOY_ROOT ?: env.PROD_DEPLOY_ROOT ?: '/alien_produ/java').trim()
+                    def prodLocalRoot = useRemoteProd ? "${env.WORKSPACE}/.jenkins_remote_prod_staging".replace('\\', '/') : (env.PROD_DEPLOY_ROOT ?: '/alien_produ/java')
+                    def sshQE = prodSsh.replace("'", "'\\''")
+                    def remQE = prodRemoteRoot.replace("'", "'\\''")
+
                     def hostEnvFile = (params.PROD_DOCKER_ENV_FILE ?: '').trim()
                     def jpwStr = jasyptParamToPlain(params.JASYPT_ENCRYPTOR_PASSWORD)
                     def writtenJasyptEnv = ''
@@ -561,8 +624,17 @@ pipeline {
                         writeFile file: writtenJasyptEnv, text: "JASYPT_ENCRYPTOR_PASSWORD=${(jpwStr)}\n", encoding: 'UTF-8'
                         effDockerEnvFile = writtenJasyptEnv
                     }
+                    if (useRemoteProd && hostEnvFile && !params.DRY_RUN) {
+                        def pulledEnv = "${env.WORKSPACE}/.jenkins_remote_pulled_env_${System.currentTimeMillis()}"
+                        def hfEsc = hostEnvFile.replace("'", "'\\''")
+                        sh """
+                            set -e
+                            scp -o BatchMode=yes -o StrictHostKeyChecking=accept-new '${sshQE}:${hfEsc}' '${pulledEnv}'
+                        """
+                        effDockerEnvFile = pulledEnv
+                    }
                     def jpfTrim = (params.PROD_JASYPT_PASSWORD_FILE ?: '').trim()
-                    def jasyptDefaultPath = "${env.PROD_DEPLOY_ROOT}/.jasypt-encryptor-password"
+                    def jasyptDefaultPath = useRemoteProd ? "${prodRemoteRoot}/.jasypt-encryptor-password" : "${env.PROD_DEPLOY_ROOT}/.jasypt-encryptor-password"
                     // 须在 for/sh 外定义:Jenkins CPS 对 sh 字符串插值只认 script 块顶层/同层 Binding 中的变量
                     def jasyptHostFile = jpfTrim ? jpfTrim : jasyptDefaultPath
                     def hasJpwPlain = jpwStr.length() > 0 ? '1' : '0'
@@ -570,11 +642,37 @@ pipeline {
                     def prodJreForJasypt = params.PROD_JRE_IMAGE ?: 'my-openjdk8-ffmpeg:v1'
 
                     if (!params.DRY_RUN) {
-                        sh """
+                        def effEsc = (effDockerEnvFile ?: '').replace("'", "'\\''")
+                        def jhfEsc = jasyptHostFile.toString().replace("'", "'\\''")
+                        if (useRemoteProd) {
+                            sh """
                             set +e
                             JENKINS_HAS_JPW=${hasJpwPlain}
                             OK=0
-                            if [ -n '${effDockerEnvFile}' ] && [ -f '${effDockerEnvFile}' ]; then OK=1; fi
+                            if [ -n '${effEsc}' ] && [ -f '${effEsc}' ]; then OK=1; fi
+                            if [ "\$JENKINS_HAS_JPW" = '1' ]; then OK=1; fi
+                            if [ "\$OK" -ne 1 ]; then
+                              if ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new '${sshQE}' "test -f '${jhfEsc}'"; then OK=1; fi
+                            fi
+                            if [ "\$OK" -ne 1 ]; then
+                              echo "ERROR: 未检测到任何可用的 Jasypt 主密码来源(远程:已 scp 的 env、构建参数 JASYPT_ENCRYPTOR_PASSWORD、或远程文件 ${jasyptDefaultPath} 至少其一须可用)"
+                              exit 1
+                            fi
+                            if [ -n '${effEsc}' ] && [ -f '${effEsc}' ] && command -v iconv >/dev/null 2>&1; then
+                              if ! iconv -f UTF-8 -t UTF-8 < '${effEsc}' >/dev/null 2>&1; then
+                                echo "ERROR: PROD_DOCKER_ENV_FILE(已 scp 到 Jenkins 临时路径)不是合法 UTF-8"
+                                exit 1
+                              fi
+                            fi
+                            echo ">>> Jasypt 来源校验通过(远程)"
+                            exit 0
+                        """
+                        } else {
+                            sh """
+                            set +e
+                            JENKINS_HAS_JPW=${hasJpwPlain}
+                            OK=0
+                            if [ -n '${effEsc}' ] && [ -f '${effEsc}' ]; then OK=1; fi
                             if [ "\$JENKINS_HAS_JPW" = '1' ]; then OK=1; fi
                             JHF='${(jasyptHostFile)}'
                             if [ -f "\$JHF" ]; then OK=1; fi
@@ -588,8 +686,8 @@ pipeline {
                                 fi
                               fi
                             fi
-                            if [ -n '${effDockerEnvFile}' ] && [ -f '${effDockerEnvFile}' ] && command -v iconv >/dev/null 2>&1; then
-                              if ! iconv -f UTF-8 -t UTF-8 < '${effDockerEnvFile}' >/dev/null 2>&1; then
+                            if [ -n '${effEsc}' ] && [ -f '${effEsc}' ] && command -v iconv >/dev/null 2>&1; then
+                              if ! iconv -f UTF-8 -t UTF-8 < '${effEsc}' >/dev/null 2>&1; then
                                 echo "ERROR: PROD_DOCKER_ENV_FILE 不是合法 UTF-8,Docker --env-file 会拒绝(常见原因:Windows 记事本或 GBK 保存)。"
                                 echo "处理:用 UTF-8 重存该文件,或 Linux: iconv -f GB18030 -t UTF-8 '${effDockerEnvFile}' > /tmp/e.utf8 && mv /tmp/e.utf8 '${effDockerEnvFile}'"
                                 echo "勿在 env 文件中使用非 UTF-8 中文注释;可用仓库 docs/jenkins/env.produ-docker.example(全 ASCII 注释)。"
@@ -608,10 +706,11 @@ pipeline {
                             echo ">>> Jasypt 来源校验通过(env-file / 构建密码 / 密钥文件至少其一有效)"
                             exit 0
                         """
+                        }
                     }
 
                     try {
-                        def composeFileForJob = (params.PROD_COMPOSE_FILE ?: '').trim() ?: "${env.PROD_DEPLOY_ROOT}/docker-compose.yml"
+                        def composeFileForJob = (params.PROD_COMPOSE_FILE ?: '').trim() ?: "${prodRemoteRoot}/docker-compose.yml"
                         def jicForJob = (params.PROD_JAR_IN_CONTAINER ?: 'app.jar').trim() ?: 'app.jar'
                         def effEnvPathForSh = (effDockerEnvFile ?: '')
                         def dcpExplicitPath = (params.PROD_DOCKER_COMPOSE_PATH ?: '').trim()
@@ -622,6 +721,9 @@ pipeline {
                         def dcpDindParam = dcpDindT ? dcpDindT : 'docker:24.0.9-cli'
                         def dcpDindOff = dcpDindParam.equalsIgnoreCase('off') || dcpDindParam == '0'
                         def dcpDindSh = dcpDindOff ? '' : dcpDindParam.replace("'", "'\\''")
+                        if (useRemoteProd) {
+                            dcpDindSh = ''
+                        }
                         def extNRaw = params.PROD_DOCKER_CREATE_EXTERNAL_NETWORKS
                         def extNOff = extNRaw != null && (extNRaw.toString().trim().equalsIgnoreCase('off') || extNRaw.toString().trim() == '0' || extNRaw.toString().trim().equalsIgnoreCase('none'))
                         def extNList = extNOff ? '' : ((extNRaw == null || extNRaw.toString().trim() == '') ? 'common-network-produ' : extNRaw.toString().trim())
@@ -630,14 +732,16 @@ pipeline {
                         for (def s in selected) {
                             def jar = "${s.module}-1.0.0.jar"
                             def srcJar = "${env.UAT_DEPLOY_ROOT}/${s.uatDir}/${(jar)}"
-                            def dstDir = "${env.PROD_DEPLOY_ROOT}/${s.prodDir}"
-                            def logHostDir = "${env.PROD_DEPLOY_ROOT}/logs/${s.prodDir}"
+                            def dstDir = "${prodLocalRoot}/${s.prodDir}"
+                            def logHostDir = useRemoteProd ? "${prodRemoteRoot}/logs/${s.prodDir}" : "${env.PROD_DEPLOY_ROOT}/logs/${s.prodDir}"
                             def srcLib = "${env.UAT_DEPLOY_ROOT}/${s.uatDir}/lib"
                             def dstLib = "${dstDir}/lib"
                             def cPrimary = s.container
                             def cFallback = s.prodDir
                             def composeSvc = s.composeService
                             def jarHostPath = "${dstDir}/${(jar)}"
+                            def jarOnProdServer = "${prodRemoteRoot}/${s.prodDir}/${(jar)}"
+                            def jarOnProdServerSh = jarOnProdServer.replace("'", "'\\''")
                             def jarHostPathSh = jarHostPath.replace("'", "'\\''")
                             def srcLibSh = srcLib.toString().replace("'", "'\\''")
                             def dstLibSh = dstLib.toString().replace("'", "'\\''")
@@ -653,17 +757,29 @@ pipeline {
                             def payCertHost = (params.PROD_PAY_CERT_HOST ?: '').trim()
                             def payCertProdDirs = ['store', 'store-platform', 'lawyer', 'dining'] as Set
                             def payCertSvc = payCertProdDirs.contains(s.prodDir) ? '1' : '0'
-                            def dstPayCertDir = "${dstDir}/alien/aliPayCert"
+                            def dstPayCertDirInspect = useRemoteProd ? "${prodRemoteRoot}/${s.prodDir}/alien/aliPayCert" : "${dstDir}/alien/aliPayCert"
                             // payCertHost 不得写进 promoteShell 的 GString(含 PAY_MOUNT="..."),CPS 会报 illegal $;内层 if/elif 整段在 Groovy 中拼接
                             def payCertHostShellEsc = (payCertHost ?: '').toString().replace('"', '\\"')
-                            def payCertPayMountInner = "                                    if [ -n \"" + payCertHostShellEsc + "\" ]; then\n" +
-                                    "                                        PAY_MOUNT=\"" + payCertHostShellEsc + "\"\n" +
-                                    '                                    elif [ -d "$JENKINS_PAY_CERT_DIR" ] && [ -n "$(ls -A "$JENKINS_PAY_CERT_DIR" 2>/dev/null)" ]; then\n' +
-                                    '                                        PAY_MOUNT="$JENKINS_PAY_CERT_DIR"\n' +
-                                    '                                    elif [ -f "$PROMOTE_DST_DIR/alien/apiclient_key.pem" ]; then\n' +
-                                    '                                        PAY_MOUNT="$PROMOTE_DST_DIR/alien"\n' +
-                                    '                                        echo ">>> WARN: 证书在 $PROMOTE_DST_DIR/alien/ 根目录(无 alien/aliPayCert 子目录),将该目录挂为容器内 /usr/local/alien/aliPayCert"\n' +
-                                    '                                    fi\n'
+                            def payCertPayMountInner
+                            if (useRemoteProd) {
+                                payCertPayMountInner = "                                    if [ -n \"" + payCertHostShellEsc + "\" ]; then\n" +
+                                        "                                        PAY_MOUNT=\"" + payCertHostShellEsc + "\"\n" +
+                                        '                                    elif [ -n "$JENKINS_PROD_SSH" ] && ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$JENKINS_PROD_SSH" "test -d \"$JENKINS_PAY_CERT_DIR\""; then\n' +
+                                        '                                        PAY_MOUNT="$JENKINS_PAY_CERT_DIR"\n' +
+                                        '                                    elif [ -n "$JENKINS_PROD_SSH" ] && ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$JENKINS_PROD_SSH" "test -f \"$JENKINS_REMOTE_PROD_ROOT/$PROMOTE_PROD_DIR/alien/apiclient_key.pem\""; then\n' +
+                                        '                                        PAY_MOUNT="$JENKINS_REMOTE_PROD_ROOT/$PROMOTE_PROD_DIR/alien"\n' +
+                                        '                                        echo ">>> WARN: 证书在远端 $PROMOTE_PROD_DIR/alien/ 根目录,将该目录挂为容器内 /usr/local/alien/aliPayCert"\n' +
+                                        '                                    fi\n'
+                            } else {
+                                payCertPayMountInner = "                                    if [ -n \"" + payCertHostShellEsc + "\" ]; then\n" +
+                                        "                                        PAY_MOUNT=\"" + payCertHostShellEsc + "\"\n" +
+                                        '                                    elif [ -d "$JENKINS_PAY_CERT_DIR" ] && [ -n "$(ls -A "$JENKINS_PAY_CERT_DIR" 2>/dev/null)" ]; then\n' +
+                                        '                                        PAY_MOUNT="$JENKINS_PAY_CERT_DIR"\n' +
+                                        '                                    elif [ -f "$PROMOTE_DST_DIR/alien/apiclient_key.pem" ]; then\n' +
+                                        '                                        PAY_MOUNT="$PROMOTE_DST_DIR/alien"\n' +
+                                        '                                        echo ">>> WARN: 证书在 $PROMOTE_DST_DIR/alien/ 根目录(无 alien/aliPayCert 子目录),将该目录挂为容器内 /usr/local/alien/aliPayCert"\n' +
+                                        '                                    fi\n'
+                            }
                             def dryRunPayCertEcho = '                                echo "[DRY_RUN] 证书卷:优先 PROD_PAY_CERT_HOST=' + (payCertHost ? payCertHost : '(未填)') + ',否则若目录存在非空则 $JENKINS_PAY_CERT_DIR 到 /usr/local/alien/aliPayCert:ro"'
                             // jasyptHostFile / effDockerEnvFile 在 promoteShell 的 GString 中接中文全角「(」等时 CPS 报 illegal $
                             def jhPath = jasyptHostFile.toString()
@@ -701,7 +817,7 @@ pipeline {
                             def jarBasenameSh = jar.replace("'", "'\\''")
                             def cPrimarySh = cPrimary.toString().replace("'", "'\\''")
                             def cFallbackSh = cFallback.toString().replace("'", "'\\''")
-                            def dstPayCertDirSh = dstPayCertDir.replace("'", "'\\''")
+                            def dstPayCertDirSh = dstPayCertDirInspect.replace("'", "'\\''")
                             def dstDirSh = dstDir.replace("'", "'\\''")
                             def uatDeployRoot = env.UAT_DEPLOY_ROOT ?: ''
                             def uatDeployRootSh = uatDeployRoot.toString().replace("'", "'\\''")
@@ -716,7 +832,9 @@ pipeline {
                             def dryRunCpEcho = "                                echo \"[DRY_RUN] cp -f " + srcJar + " \$PROMOTE_DST_DIR/\""
                             def cpSrcToDst = "                                cp -f '" + srcJarSh + "' " + "\$" + "PROMOTE_DST_DIR/"
                             def echoJarLine = '                            echo ">>> [jar] ' + s.module + ': ' + s.uatDir + ' -> ' + s.prodDir + '(compose 服务名: $COMPOSE_SVC, 容器内 jar: $JCP)"'
-                            def dryRunLogDirEcho = "                                echo \"[DRY_RUN] mkdir -p " + logHostDir + "(日志目录,与 compose ./logs/" + s.prodDir + ":/app/logs 一致)\""
+                            def dryRunLogDirEcho = useRemoteProd ?
+                                    '                                echo "[DRY_RUN] mkdir -p 本地: $(dirname $PROMOTE_DST_DIR)/logs/$PROMOTE_PROD_DIR -> rsync -> 生产: ' + logHostDir + '"' :
+                                    "                                echo \"[DRY_RUN] mkdir -p " + logHostDir + "(日志目录,与 compose ./logs/" + s.prodDir + ":/app/logs 一致)\""
                             def jasyptExplLine = "                            JENKINS_JASYPT_EXPL=" + jasyptExplicit
                             // 无引号 JENKINS_*= 赋值不得写进 promoteShell 的 GString,否则如 ${autoCreate} 会触发非法 $ 转义
                             def jenkinsEnvBools = "                            JENKINS_DRY_RUN=" + dryRunSh + "\n" +
@@ -729,6 +847,9 @@ pipeline {
                                     "                            DCP_V2_PLUGIN='" + dcpV2ShellQuoted + "'\n" +
                                     "                            DCP_DIND_IMAGE='" + dcpDindSh + "'"
                             def dcpExtNetsLine = "                            DCP_EXT_NETS='" + extNetsSh + "'"
+                            def jarOnProdExportSh = useRemoteProd ? jarOnProdServerSh : ''
+                            def sshExport = useRemoteProd ? sshQE : ''
+                            def remExport = useRemoteProd ? remQE : ''
                             // export 与环境变量行整段用 + 拼接;大 GString 中 ANY 'VAR=''${...}'' 均可能被 CPS 报 “illegal $”(与 eff* / ext* 等标识符相关)
                             def promoteExportBlock = "                            export JENKINS_HOME='" + jenkinsHomeSh + "'\n" +
                                     "                            COMPOSE_FILE='" + composeFileForJob + "'\n" +
@@ -736,6 +857,9 @@ pipeline {
                                     "                            PROMOTE_PROD_DIR='" + promoteProdDirSh + "'\n" +
                                     "                            PROMOTE_DST_DIR='" + dstDirSh + "'\n" +
                                     "                            JAR_HOST='" + jarHostPathSh + "'\n" +
+                                    "                            JAR_ON_PROD_SERVER='" + jarOnProdExportSh + "'\n" +
+                                    "                            JENKINS_PROD_SSH='" + sshExport + "'\n" +
+                                    "                            JENKINS_REMOTE_PROD_ROOT='" + remExport + "'\n" +
                                     "                            PROMOTE_SRC_LIB='" + srcLibSh + "'\n" +
                                     "                            PROMOTE_DST_LIB='" + dstLibSh + "'\n" +
                                     "                            JAR_BASENAME='" + jarBasenameSh + "'\n" +
@@ -748,9 +872,15 @@ pipeline {
                                     "                            JCP_ALT='" + jcpAltEscaped + "'\n" +
                                     "                            JENKINS_PROD_ENV_FILE='" + effEnvPathForSh + "'\n"
 
+                            def remoteDockerShimBlock = useRemoteProd ? '\n                            if [ -n "$JENKINS_PROD_SSH" ]; then\n                              docker() { ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$JENKINS_PROD_SSH" sudo docker "$@"; }\n                            fi\n' : '\n'
+                            def rsyncRemoteBlock = useRemoteProd ? '\n                                if [ -n "$JENKINS_PROD_SSH" ]; then\n                                    echo ">>> rsync 制品到 $JENKINS_PROD_SSH:$JENKINS_REMOTE_PROD_ROOT/$PROMOTE_PROD_DIR/"\n                                    ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$JENKINS_PROD_SSH" "sudo mkdir -p $JENKINS_REMOTE_PROD_ROOT/$PROMOTE_PROD_DIR $JENKINS_REMOTE_PROD_ROOT/logs/$PROMOTE_PROD_DIR"\n                                    rsync -az "$PROMOTE_DST_DIR/" "$JENKINS_PROD_SSH:$JENKINS_REMOTE_PROD_ROOT/$PROMOTE_PROD_DIR/"\n                                    rsync -az "$(dirname "$PROMOTE_DST_DIR")/logs/$PROMOTE_PROD_DIR/" "$JENKINS_PROD_SSH:$JENKINS_REMOTE_PROD_ROOT/logs/$PROMOTE_PROD_DIR/" 2>/dev/null || true\n                                fi\n' : '\n'
+                            def composeExistsBlockSh = useRemoteProd ? '\n                                if ! ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$JENKINS_PROD_SSH" "test -f $COMPOSE_FILE"; then\n                                    echo "ERROR: 远程未找到 docker compose 文件: $COMPOSE_FILE"\n                                    exit 1\n                                fi\n' : '\n                                if [ ! -f "$COMPOSE_FILE" ]; then\n                                    echo "ERROR: 未找到 docker compose 文件: $COMPOSE_FILE。请将 PROD_COMPOSE_FILE/PROD_DEPLOY_ROOT 与现网 yml 对齐,例如 /alien_produ/java/docker-compose.yml"\n                                    exit 1\n                                fi\n'
+                            def composeDirBlockSh = useRemoteProd ? '                                COMPOSE_DIR=$(dirname "$COMPOSE_FILE")' : '                                COMPOSE_DIR=$(cd "$(dirname "$COMPOSE_FILE")" && pwd)'
+
                             def promoteShell = """
                             set -e
                             #JENKINS_PLACE_EXPORT_ENV
+                            #JENKINS_PLACE_REMOTE_DOCKER_SHIM
                             #JENKINS_PLACE_DCP_THREE
                             #JENKINS_PLACE_DCP_EXT_NETS
                             #JENKINS_PLACE_ENV_BOOLS
@@ -779,6 +909,9 @@ pipeline {
                                 if [ -d "\$PROMOTE_SRC_LIB" ]; then
                                     echo "[DRY_RUN] 同步 lib: \$PROMOTE_SRC_LIB -> \$PROMOTE_DST_LIB"
                                 fi
+                                if [ -n "\$JENKINS_PROD_SSH" ]; then
+                                    echo "[DRY_RUN] rsync \$PROMOTE_DST_DIR/ -> \$JENKINS_PROD_SSH:\$JENKINS_REMOTE_PROD_ROOT/\$PROMOTE_PROD_DIR/"
+                                fi
                                 echo "[DRY_RUN] 将执行: docker compose -f \$COMPOSE_FILE --project-directory \$COMPOSE_DIR(restart 或 up -d --no-deps)\$COMPOSE_SVC;非 DRY 时脚本会设 COMPOSE_DIR"
                                 if [ "\$JENKINS_PAY_CERT_SVC" = '1' ]; then
                                     #JENKINS_PLACE_DRYRUN_PAYCERT_ECHO
@@ -799,7 +932,7 @@ pipeline {
                                     fi
                                 fi
                                 mkdir -p "\$PROMOTE_DST_DIR"
-                                mkdir -p "\$JENKINS_LOG_DIR"
+                                mkdir -p "\$(dirname "\$PROMOTE_DST_DIR")/logs/\$PROMOTE_PROD_DIR"
                             #JENKINS_PLACE_CP_PROD
                                 if [ -d "\$PROMOTE_SRC_LIB" ]; then
                                     rm -rf "\$PROMOTE_DST_LIB"
@@ -817,11 +950,10 @@ pipeline {
                                     echo "ERROR: jar 不是有效 zip/jar(可能拷贝不完整或源文件损坏): \$JAR_HOST"
                                     exit 1
                                 fi
-                                if [ ! -f "\$COMPOSE_FILE" ]; then
-                                    echo "ERROR: 未找到 docker compose 文件: \$COMPOSE_FILE。请将 PROD_COMPOSE_FILE/PROD_DEPLOY_ROOT 与现网 yml 对齐,例如 /alien_produ/java/docker-compose.yml"
-                                    exit 1
-                                fi
-                                COMPOSE_DIR=\$(cd "\$(dirname "\$COMPOSE_FILE")" && pwd)
+                                #JENKINS_PLACE_RSYNC_REMOTE
+                                #JENKINS_PLACE_COMPOSE_EXISTS
+                                #JENKINS_PLACE_COMPOSE_DIR
+                                JAR_EXPECT="\${JAR_ON_PROD_SERVER:-\$JAR_HOST}"
                                 dcp_ensure_external_nets() {
                                     if [ -z "\$DCP_EXT_NETS" ]; then
                                         return 0
@@ -1068,10 +1200,10 @@ pipeline {
                                     FMT_MNT=\$(printf '{{range .Mounts}}{{if eq .Destination "%s"}}{{.Source}}{{end}}{{end}}' "\$JCP")
                                     ACTUAL_SRC=\$(docker inspect -f "\$FMT_MNT" "\$CNAME" 2>/dev/null | head -1)
                                     MISMATCH=0
-                                    if [ -n "\$ACTUAL_SRC" ] && [ "\$ACTUAL_SRC" != "\$JAR_HOST" ]; then
+                                    if [ -n "\$ACTUAL_SRC" ] && [ "\$ACTUAL_SRC" != "\$JAR_EXPECT" ]; then
                                         if command -v realpath >/dev/null 2>&1; then
                                             R1=\$(realpath "\$ACTUAL_SRC" 2>/dev/null || echo "\$ACTUAL_SRC")
-                                            R2=\$(realpath "\$JAR_HOST" 2>/dev/null || echo "\$JAR_HOST")
+                                            R2=\$(realpath "\$JAR_EXPECT" 2>/dev/null || echo "\$JAR_EXPECT")
                                             [ "\$R1" != "\$R2" ] && MISMATCH=1
                                         else
                                             MISMATCH=1
@@ -1079,12 +1211,12 @@ pipeline {
                                     fi
                                     if [ "\$MISMATCH" -eq 1 ] && [ -n "\$ACTUAL_SRC" ]; then
                                         if [ "\$JENKINS_AUTO_RECREATE" = 'true' ]; then
-                                            echo ">>> WARN: 容器 \$CNAME 的 \$JCP 挂载源与本次制品路径不一致,将删除并 compose 重建(挂载源=\$ACTUAL_SRC,本次=\$JAR_HOST)"
+                                            echo ">>> WARN: 容器 \$CNAME 的 \$JCP 挂载源与本次制品路径不一致,将删除并 compose 重建(挂载源=\$ACTUAL_SRC,本次=\$JAR_EXPECT)"
                                             docker rm -f "\$CNAME"
                                             CNAME=""
                                             RECREATE=1
                                         else
-                                            echo "ERROR: 容器 \$CNAME 的 \$JCP 挂载源为 \$ACTUAL_SRC,本次流水线 jar 为 \$JAR_HOST。"
+                                            echo "ERROR: 容器 \$CNAME 的 \$JCP 挂载源为 \$ACTUAL_SRC,本次流水线 jar 为 \$JAR_EXPECT。"
                                             echo "仅 restart 不会更改 bind mount。请执行 docker rm 后 dcp up,或开启 AUTO_RECREATE_IF_MOUNT_MISMATCH。"
                                             exit 1
                                         fi
@@ -1295,11 +1427,11 @@ pipeline {
                                     fi
                                     FMT2=\$(printf '{{range .Mounts}}{{if eq .Destination "%s"}}{{.Source}}{{end}}{{end}}' "\$JCP_RESOLVED")
                                     ACTUAL_SRC2=\$(docker inspect -f "\$FMT2" "\$CNAME" 2>/dev/null | head -1)
-                                    echo ">>> 单文件型挂载校验: Dest=\$JCP_RESOLVED -> Source=\${ACTUAL_SRC2:-(无/或为目录挂 /app 属正常)} 期望宿主机 jar: \$JAR_HOST"
-                                    if [ -n "\$ACTUAL_SRC2" ] && [ "\$ACTUAL_SRC2" != "\$JAR_HOST" ]; then
+                                    echo ">>> 单文件型挂载校验: Dest=\$JCP_RESOLVED -> Source=\${ACTUAL_SRC2:-(无/或为目录挂 /app 属正常)} 期望宿主机 jar: \$JAR_EXPECT"
+                                    if [ -n "\$ACTUAL_SRC2" ] && [ "\$ACTUAL_SRC2" != "\$JAR_EXPECT" ]; then
                                         if command -v realpath >/dev/null 2>&1; then
                                             R1=\$(realpath "\$ACTUAL_SRC2" 2>/dev/null || echo "\$ACTUAL_SRC2")
-                                            R2=\$(realpath "\$JAR_HOST" 2>/dev/null || echo "\$JAR_HOST")
+                                            R2=\$(realpath "\$JAR_EXPECT" 2>/dev/null || echo "\$JAR_EXPECT")
                                             if [ "\$R1" != "\$R2" ]; then
                                                 echo "ERROR: 单文件卷宿主机 Source 与本次 cp 的 jar 仍不一致"
                                                 exit 1
@@ -1358,6 +1490,10 @@ pipeline {
                             sh( promoteShell.toString()
                             // replaceAll 的替换串含 shell 的 $CNAME 等,Java 会当「组引用」;须 quoteReplacement
                                     .replaceAll(~/\n[ \t]*#JENKINS_PLACE_EXPORT_ENV\n/, java.util.regex.Matcher.quoteReplacement("\n" + promoteExportBlock))
+                                    .replaceAll(~/\n[ \t]*#JENKINS_PLACE_REMOTE_DOCKER_SHIM\n/, java.util.regex.Matcher.quoteReplacement(remoteDockerShimBlock))
+                                    .replaceAll(~/\n[ \t]*#JENKINS_PLACE_RSYNC_REMOTE\n/, java.util.regex.Matcher.quoteReplacement(rsyncRemoteBlock))
+                                    .replaceAll(~/\n[ \t]*#JENKINS_PLACE_COMPOSE_EXISTS\n/, java.util.regex.Matcher.quoteReplacement(composeExistsBlockSh))
+                                    .replaceAll(~/\n[ \t]*#JENKINS_PLACE_COMPOSE_DIR\n/, java.util.regex.Matcher.quoteReplacement("\n" + composeDirBlockSh + "\n"))
                                     .replaceAll(~/\n[ \t]*#JENKINS_PLACE_DRYRUN_PAYCERT_ECHO\n/, java.util.regex.Matcher.quoteReplacement("\n" + dryRunPayCertEcho + "\n"))
                                     .replaceAll(~/\n[ \t]*#JENKINS_PLACE_DRYRUN_EFF\n/, java.util.regex.Matcher.quoteReplacement("\n" + dryRunEffIfFi + "\n"))
                                     .replaceAll(~/\n[ \t]*#JENKINS_PLACE_DRYRUN_JHF_D\n/, java.util.regex.Matcher.quoteReplacement("\n" + jasyptDryHfDBlock + "\n"))
@@ -1399,7 +1535,7 @@ pipeline {
         failure {
             echo '>>> 流水线失败:优先检查 UAT 是否已发版、UAT_DEPLOY_ROOT/PROD_DEPLOY_ROOT、compose 路径、Docker 与挂载'
             echo '>>> 制品与 jar 来自 UAT 目录(UAT_DEPLOY_ROOT/<uatDir>/),不依赖 Git;仅当「BOOTSTRAP_SYNC_MODE=from_git」才会拉取 Git 取 bootstrap-prod.yml。默认 from_jar=从 UAT jar 解压,无需 Git'
-            echo '>>> 若 Jenkins 与生产 Docker 不在同一台机器,本 Job 无法改远程宿主机上的容器;需在生产 dockerd 所在宿主机上核对挂载与 jar,或在该机执行晋升/重建'
+            echo '>>> 若 Jenkins 与生产 Docker 不在同一台机器:须配置 PROD_SSH_TARGET(免密 ssh + 目标机 sudo docker);制品经 rsync 到对方 PROD_DEPLOY_ROOT;留空 PROD_SSH_TARGET 则仍为本机 docker compose'
             echo '>>> 若 source / compose 报 invalid env file / invalid utf8:将 .env.produ-docker 改为 UTF-8,或用 env.produ-docker.example;Jasypt 需进入容器 env,检查 compose、.env 与 dcp 前 set -a、AUTO_RECREATE'
             echo '>>> 若报 unknown shorthand flag: f 或本机无 compose:默认已用 PROD_DOCKER_COMPOSE_DIND_IMAGE(如 docker:24.0.9-cli)通过 docker run 回退;失败则检查能 pull 该镜像、/var/run/docker.sock、或设 PROD_DOCKER_COMPOSE_PATH/PROD_DOCKER_COMPOSE_V2_PLUGIN/挂 cli-plugins'
             echo '>>> 若报 network ... declared as external, but could not be found:设 PROD_DOCKER_CREATE_EXTERNAL_NETWORKS 为 yml 中网名(默认预建 common-network-produ),或宿主机 docker network create'