星尘

GitLab流水线配置

发表于 分类于 ，

现在有两套 Harbor 仓库，两套仓库的镜像不能交叉推送。算法仓库一套代码需要打出十几个镜像，本次借助 gitlab 来实现不同环境的镜像构建和推送，并且到指定机器进行拉取。

安装/注册 GitLab-Runner

略，可参考 qwen 等大模型的答案，或参考 Install GitLab Runner using the official GitLab repositories | GitLab Docs

GitLab-Runner 配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
linux:~$ cat /etc/gitlab-runner/config.toml 
concurrent = 1
check_interval = 0
shutdown_timeout = 0

[session_server]
  session_timeout = 1800

[[runners]]
  name = "runner名称"  # 当前 runner 的名称
  url = "https://gitlab.domain.com"  # 要连接的 gitlab 的地址
  id = 8
  token = "glrt-xxxx"  # token
  token_obtained_at = 2025-10-15T01:49:25Z
  token_expires_at = 0001-01-01T00:00:00Z
  executor = "docker"   # 执行器，可选的还有 shell，kubernetes，ssh.......
  [runners.cache]
    MaxUploadedArchiveSize = 0
    [runners.cache.s3]
    [runners.cache.gcs]
    [runners.cache.azure]
  [runners.docker]
    tls_verify = false
    image = "alpine:latest"  # 默认镜像，可在配置文件中覆盖
    volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/cache"]  # 挂载本地 docker sock 到容器中
    pull_policy = "if-not-present"  # 镜像不存在才拉取
    privileged = false
    disable_entrypoint_overwrite = false
    oom_kill_disable = false
    disable_cache = false
    shm_size = 0
    network_mtu = 0

gitlab-ci.yml 配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
workflow:
  rules:
    # 推送到 master or ci/test-pipeline 才触发
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "ci/test-pipeline"'

stages:
  - build
  - pull

variables:
  ENV: "yz"  # yz 或 by（环境的定义，默认为 yz）

setup-env:
  stage: .pre
  image: docker:24.0.5  # 使用的镜像
  variables:
    GIT_STRATEGY: none  # 不克隆代码
  tags:
    - yz_192.168.6.60   # 在哪一台机器上执行
  script:
    - echo "FEISHU_WEBHOOK=$FEISHU_WEBHOOK" >> env.env
    - |
      case "$ENV" in
        "by")
          # 拉取机器1 (192.168.183.50)
          echo "PULL1_HOST=192.168.183.50" >> env.env
          echo "PULL1_USER=$SSH_USER_BY_PULL1" >> env.env
          echo "PULL1_PASS=$SSH_PASS_BY_PULL1" >> env.env

          # 拉取机器2 (192.168.183.51)
          echo "PULL2_HOST=192.168.183.51" >> env.env
          echo "PULL2_USER=$SSH_USER_BY_PULL2" >> env.env
          echo "PULL2_PASS=$SSH_PASS_BY_PULL2" >> env.env
          
          # 打包，推送，拉取脚本
          echo 'BUILD_SCRIPT=./deploy/gitlab_ci_build_image.sh' >> env.env
          echo 'PUSH_SCRIPT=./deploy/gitlab_ci_by_push_image.sh' >> env.env
          echo 'PULL_SCRIPT=./deploy/gitlab_ci_by_pull_image.sh' >> env.env
          ;;
        "yz")
          # 拉取机器1 (192.168.6.63)
          echo "PULL1_HOST=192.168.6.63" >> env.env
          echo "PULL1_USER=$SSH_USER_YZ_PULL1" >> env.env
          echo "PULL1_PASS=$SSH_PASS_YZ_PULL1" >> env.env

          # 拉取机器2 (192.168.6.65)
          echo "PULL2_HOST=192.168.6.65" >> env.env
          echo "PULL2_USER=$SSH_USER_YZ_PULL2" >> env.env
          echo "PULL2_PASS=$SSH_PASS_YZ_PULL2" >> env.env
          
           # 打包，推送，拉取脚本
          echo 'BUILD_SCRIPT=./deploy/gitlab_ci_build_image.sh' >> env.env
          echo 'PUSH_SCRIPT=./deploy/gitlab_ci_yz_push_image.sh' >> env.env
          echo 'PULL_SCRIPT=./deploy/gitlab_ci_yz_pull_image.sh' >> env.env
          ;;
        *)
          echo "❌ ERROR: Invalid ENV='$ENV'. Must be 'by' or 'yz'."
          exit 1
          ;;
      esac
      echo "✅ Environment configured for $ENV"
      cat env.env
  artifacts:
    reports:
      dotenv: env.env

build-and-push-yz:
  stage: build
  image: docker:24.0.5
  before_script:
    - apk add --no-cache git bash sshpass openssh-client
    - echo "=========== 拉取子仓库 ==========="
    - git config --global url."https://oauth2:${SUBMODULE_TOKEN}@gitlab.ultra-ai.app/".insteadOf "https://gitlab.ultra-ai.app/"
    - git submodule sync --recursive
    - git submodule update --init --recursive
  tags:
    - yz_192.168.6.60
  script:
    - |
      echo "=========== 输出主仓库 git log 信息 ==========="
      git log -5
      echo "=========== 输出子仓库 git log 信息 ==========="
      cd ultralytics_local && git log -5 && cd ..
      ls -la && du -sh *
      chmod +x deploy/gitlab_ci_build_image.sh &&  bash ./deploy/gitlab_ci_build_image.sh
      chmod +x deploy/gitlab_ci_yz_push_image.sh &&  bash ./deploy/gitlab_ci_yz_push_image.sh
  environment:
    name: $ENV/build
  rules:
    - if: '$ENV == "yz"'

pull-to-yz:
  stage: pull
  image: docker:24.0.5
  tags:
    - yz_192.168.6.60
  before_script:
    - apk add --no-cache openssh-client sshpass
  script:
    - |
      echo "=========== 更新机器: $PULL1_HOST  ==========="
      export SSHPASS="$PULL1_PASS"
      sshpass -e scp -o StrictHostKeyChecking=no ./deploy/gitlab_ci_yz_pull_image.sh "$PULL1_USER@$PULL1_HOST:/tmp/"
      sshpass -e ssh -o StrictHostKeyChecking=no "$PULL1_USER@$PULL1_HOST" "bash /tmp/gitlab_ci_yz_pull_image.sh"
      docker images | grep ultraalgorithm
      
      echo "=========== 更新机器: $PULL2_HOST ==========="
      export SSHPASS="$PULL2_PASS"
      sshpass -e scp -o StrictHostKeyChecking=no ./deploy/gitlab_ci_yz_pull_image.sh "$PULL2_USER@$PULL2_HOST:/tmp/"
      sshpass -e ssh -o StrictHostKeyChecking=no "$PULL2_USER@$PULL2_HOST" "bash /tmp/gitlab_ci_yz_pull_image.sh"
      docker images | grep ultraalgorithm
  environment:
    name: $ENV/pull1
  rules:
    - if: '$ENV == "yz"'

# ===================================================================================================================

build-and-push-by:
  stage: build
  image: docker:24.0.5
  before_script:
    - apk add --no-cache git bash sshpass openssh-client
    - echo "=========== 拉取子仓库 ==========="
    - git config --global url."https://oauth2:${SUBMODULE_TOKEN}@gitlab.ultra-ai.app/".insteadOf "https://gitlab.ultra-ai.app/"
    - git submodule sync --recursive
    - git submodule update --init --recursive
  tags:
    - by_192.168.183.53
  script:
    - |
      echo "=========== 输出主仓库 git log 信息 ==========="
      git log -5
      echo "=========== 输出子仓库 git log 信息 ==========="
      cd ultralytics_local && git log -5 && cd ..
      ls -la && du -sh *
      chmod +x deploy/gitlab_ci_build_image.sh &&  bash ./deploy/gitlab_ci_build_image.sh
      chmod +x deploy/gitlab_ci_by_push_image.sh &&  bash ./deploy/gitlab_ci_by_push_image.sh
  environment:
    name: $ENV/build
  rules:
    - if: '$ENV == "by"'

pull-to-by:
  stage: pull
  image: docker:24.0.5
  tags:
    - by_192.168.183.53
  before_script:
    - apk add --no-cache openssh-client sshpass
  script:
    - |
      echo "=========== 更新机器: $PULL1_HOST  ==========="
      export SSHPASS="$PULL1_PASS"
      sshpass -e scp -o StrictHostKeyChecking=no ./deploy/gitlab_ci_by_pull_image.sh "$PULL1_USER@$PULL1_HOST:/tmp/"
      sshpass -e ssh -o StrictHostKeyChecking=no "$PULL1_USER@$PULL1_HOST" "bash /tmp/gitlab_ci_by_pull_image.sh"
      docker images | grep ultraalgorithm
      
      echo "=========== 更新机器: $PULL2_HOST ==========="
      export SSHPASS="$PULL2_PASS"
      sshpass -e scp -o StrictHostKeyChecking=no ./deploy/gitlab_ci_by_pull_image.sh "$PULL2_USER@$PULL2_HOST:/tmp/"
      sshpass -e ssh -o StrictHostKeyChecking=no "$PULL2_USER@$PULL2_HOST" "bash /tmp/gitlab_ci_by_pull_image.sh"
      docker images | grep ultraalgorithm
  environment:
    name: $ENV/pull1
  rules:
    - if: '$ENV == "by"'

# =================================================================================================================

notify-feishu-success:
  stage: .post
  image: docker:24.0.5
  tags:
    - yz_192.168.6.60
  before_script:
    - apk add --no-cache curl jq openssl bash
  script:
    - chmod +x ./deploy/notify_feishu.sh
    - bash ./deploy/notify_feishu.sh success
  when: on_success
  environment:
    name: $ENV/notify

notify-feishu-failure:
  stage: .post
  image: docker:24.0.5
  tags:
    - yz_192.168.6.60
  before_script:
    - apk add --no-cache curl jq openssl bash
  script:
    - chmod +x ./deploy/notify_feishu.sh
    - bash ./deploy/notify_feishu.sh failed
  when: on_failure
  environment:
    name: $ENV/notify

注意事项

  1. 因为是借用宿主机的 docker 环境进行打包，所以 runner 所在的机器也有会新构建的镜像（这其实污染了环境）

  2. 使用的时候，需要在 gitlab中配置上述环境变量信息

17609505210801760950520778.png

  1. 并且确保代码仓库的 deploy 中存在如下脚本

17609503290771760950328488.png

效果

17609517150771760951714438.png

17609507490751760950748825.png