一、确认配置

两台镜像仓库，模拟生产环境高可用

节点类型	数量	CPU	内存	磁盘	主机名	IP	关键服务/组件
私有镜像库节点	2	4核	4G	40G	harbor1,harbor2	10.31.3.241-242/16; 10.31.3.250/16(VIP)	Harbor, contained, cni, nerdctl, HAProxy, Keepalived

没有说明是某个节点单独运行的，需要每个节点都执行
主要组件版本：harbor 2.13.2，containerd 1.7.27， cni 1.7.1， nerdctl 2.1.3

二、配置环境

1. 基础环境配置

# 1. 配置阿里云Ubuntu镜像源
sudo sed -i 's|archive.ubuntu.com|mirrors.aliyun.com|g' /etc/apt/sources.list
sudo sed -i 's|security.ubuntu.com|mirrors.aliyun.com|g' /etc/apt/sources.list

# 2. 安装必要依赖
sudo apt update && sudo apt upgrade -y
sudo apt install -y  chrony containerd iptables haproxy keepalived rsync

# 3. 配置时间同步，timedatectl查看当前时间/时区
sudo timedatectl set-timezone Asia/Shanghai
sudo systemctl enable chrony && sudo systemctl start chrony

# 4.1 设置主节点主机名为harbor1
sudo hostnamectl set-hostname harbor1
# 4.2 设置备节点主机名为harbor2
sudo hostnamectl set-hostname harbor2

# 5.1 harbor1编辑固定IP为10.31.3.241，也可在路由器上操作mac与IP的绑定
cat <<EOF | sudo tee /etc/netplan/50-cloud-init.yaml
network:
  version: 2
  ethernets:
    enp6s18:
      dhcp4: no
      addresses: [10.31.3.241/16]
      routes:
        - to: default
          via: 10.31.0.1
      nameservers:
        addresses: [10.31.0.1, 223.5.5.5]
EOF

# 5.2 harbor2编辑固定IP为10.31.3.242，应用网络修改运行netplan apply
cat <<EOF | sudo tee /etc/netplan/50-cloud-init.yaml
network:
  version: 2
  ethernets:
    enp6s18:
      dhcp4: no
      addresses: [10.31.3.242/16]
      routes:
        - to: default
          via: 10.31.0.1
      nameservers:
        addresses: [10.31.0.1, 223.5.5.5]
EOF

# 6. 添加主机解析，10.31.3.250为后面负载均衡虚拟的VIP地址，确认局域网内没有被占用
cat <<EOF | sudo tee -a /etc/hosts
10.31.3.241 harbor1
10.31.3.242 harbor2
10.31.3.250 harbor.mysen.top
EOF

# 7. 配置ssh启用root密码登录，方便后续传输证书
echo "PermitRootLogin yes" | sudo tee -a /etc/ssh/sshd_config
systemctl restart ssh
passwd root #设置root密码

2. ‌配置containerd‌

# 1. 生成默认配置再修改部分参数，验证安装containerd --version，重启容器systemctl restart containerd
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml
sudo sed -i 's|SystemdCgroup = false|SystemdCgroup = true|g' /etc/containerd/config.toml
sudo sed -i 's|registry.k8s.io/pause:3.8|registry.aliyuncs.com/google_containers/pause:3.10|g' /etc/containerd/config.toml

# 2. 安装加载cni网络插件（自行科学上网）
wget https://github.com/containernetworking/plugins/releases/download/v1.7.1/cni-plugins-linux-amd64-v1.7.1.tgz
sudo mkdir -p /opt/cni/bin && sudo tar -xzvf cni-plugins-linux-amd64-v1.7.1.tgz -C /opt/cni/bin

# 3. 安装配置nerdctl‌，验证安装运行nerdctl --version
wget https://github.com/containerd/nerdctl/releases/download/v2.1.3/nerdctl-2.1.3-linux-amd64.tar.gz
sudo tar Cxzvvf /usr/local/bin nerdctl-2.1.3-linux-amd64.tar.gz
sudo nerdctl network create --subnet 172.20.0.0/16 harbor-net

# 4. 创建compose，验证compose version，替代docker-compose
sudo tee /usr/local/bin/compose <<'EOF'
#!/bin/bash
nerdctl compose "$@"
EOF
sudo chmod +x /usr/local/bin/compose

# 5. 环境变量配置，添加必要别名以及创建符号链接，替代docker命令
echo "alias docker='nerdctl'" >> ~/.bashrc
echo "alias docker-compose='nerdctl compose'" >> ~/.bashrc
source ~/.bashrc
sudo ln -s /usr/local/bin/nerdctl /usr/local/bin/docker
sudo ln -s /usr/local/bin/nerdctl-compose /usr/local/bin/docker-compose

3. 生成证书

# 1. 创建证书目录并进入(两个节点都创建目录后，证书的后续2-7步操作都在harbor1节点上操作)
sudo mkdir -p /data/cert && cd /data/cert

# 2. 生成CA证书‌，OpenSSL 3.0+生成的私钥默认使用PKCS#8格式，而Harbor 2.13.2需要PKCS#1格式
openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ca.key -out ca.crt -subj "/CN=harbor.mysen.top"

# 3. ‌生成服务器证书‌
openssl req -new -newkey rsa:2048 -nodes -keyout harbor.mysen.top.key -out harbor.mysen.top.csr -subj "/CN=harbor.mysen.top"

# 4. 生成x509 v3扩展文件
cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names

[alt_names]
DNS.1=harbor.mysen.top
IP.1 = 10.31.3.250
IP.2 = 10.31.3.241
IP.3 = 10.31.3.242
EOF

# 5. 使用CA签发证书
openssl x509 -req -days 365 -extfile v3.ext -in harbor.mysen.top.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out harbor.mysen.top.crt

# 6. 合并证书为PEM格式，并设置权限，harbor.mysen.top.pem用于haproxy，harbor.pem用于harbor.yml
cat harbor.mysen.top.key harbor.mysen.top.crt > harbor.mysen.top.pem
chmod 600 harbor.mysen.top.pem
cat ca.crt harbor.mysen.top.crt > harbor.pem
chmod 600 harbor.pem

# 7. 同步证书到harbor2节点
rsync -avz /data/cert/ root@harbor2:/data/cert/

# 8. 两个节点添加CA证书信任
sudo cp /data/cert/ca.crt /usr/local/share/ca-certificates/
sudo cp /data/cert/harbor.mysen.top.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates --fresh

4. 部署Harbor

# 1. 下载Harbor离线安装包，解压并进入目录
sudo wget https://github.com/goharbor/harbor/releases/download/v2.13.2/harbor-offline-installer-v2.13.2.tgz
sudo tar xvf harbor-offline-installer-v2.13.2.tgz && cd harbor

# 2. 复制默认配置文件，修改hostname为harbor.mysen.top，修改https证书路径
cp harbor.yml.tmpl harbor.yml
sudo sed -i 's|reg.mydomain.com|harbor.mysen.top|g' /root/harbor/harbor.yml
sudo sed -i 's|/your/certificate/path|/data/cert/harbor.pem|g' /root/harbor/harbor.yml
sudo sed -i 's|/your/private/key/path|/data/cert/harbor.mysen.top.key|g' /root/harbor/harbor.yml

# 3. 使用创建的网络配置
cat <<EOF | sudo tee -a /root/harbor/harbor.yml
network:
  name: harbor-net
  external: true
EOF

# 4. 修改Harbor安装脚本，docker-compoes替换为compose，注释掉Docker检查部分
sudo sed -i 's|=docker-|=|g' /root/harbor/install.sh
sudo sed -i 's/^.*check_docker.*$/#&/' /root/harbor/install.sh
sudo sed -i 's|down|stop \&\& $DOCKER_COMPOSE rm|g' /root/harbor/install.sh

# 5. 修改prepare脚本，替换docker为nerdctl
sudo sed -i 's|docker|nerdctl|g' /root/harbor/prepare

# 6. 使用nerdctl解压生成本地镜像文件，防止在线拉取镜像
nerdctl load -i harbor.v2.13.2.tar.gz

# 7. 安装并启动Harbor，验证在局域网内其他主机上通过浏览器访问https://10.31.3.241或242
sudo ./prepare && sudo ./install.sh

5. 负载均衡

# 1. 配置HAProxy，stats auth后为账号密码
cat <<EOF | sudo tee -a /etc/haproxy/haproxy.cfg

frontend harbor-https
    bind *:443 ssl crt /data/cert/harbor.mysen.top.pem
    mode http
    option httplog
    option forwardfor
    default_backend harbor-backend

backend harbor-backend
    mode http
    balance roundrobin
    server harbor1 10.31.3.241:443 check
    server harbor2 10.31.3.242:443 check

listen stats
    bind *:8080
    mode http
    stats enable
    stats uri /haproxy?stats
    stats realm Haproxy\ Statistics
    stats auth mysen:kubernetes
    stats hide-version
    stats refresh 30s
EOF

# 2. 检查配置语法
sudo haproxy -c -f /etc/haproxy/haproxy.cfg

# 3.  启动HAProxy，验证运行systemctl status haproxy
sudo systemctl restart haproxy && sudo systemctl enable haproxy

# 4.1 配置主节点 (harbor1)的keepalived，根据实际网卡名称修改enp6s18，priority为优先级
sudo tee /etc/keepalived/keepalived.conf <<-'EOF'
global_defs {
    router_id harbor_ha_1
}

vrrp_script chk_haproxy {
    script "killall -0 haproxy"
    interval 2
    weight 2
}

vrrp_instance VI_1 {
    state MASTER
    interface enp6s18
    virtual_router_id 51
    priority 101
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass kubernetes
    }
    virtual_ipaddress {
        10.31.3.250/16
    }
    track_script {
        chk_haproxy
    }
}
EOF

# 4.2 配置备节点（harbor2）keepalived的配置文件，auth_pass后为密码，备节点的优先级低于主节点
sudo tee /etc/keepalived/keepalived.conf <<-'EOF'
global_defs {
    router_id harbor_ha_2
}

vrrp_script chk_haproxy {
    script "killall -0 haproxy"
    interval 2
    weight 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface enp6s18
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass kubernetes
    }
    virtual_ipaddress {
        10.31.3.250/16
    }
    track_script {
        chk_haproxy
    }
}
EOF

# 5. 启动Keepalived，验证运行sudo systemctl status keepalived
sudo systemctl restart keepalived && sudo systemctl enable keepalived

6. 部署验证

# 1. 完整重启服务，确保依赖服务正确启动，先停止haproxy后，进入/root/harbor/目录再执行。
sudo systemctl stop haproxy
cd /root/harbor/ && docker-compose down && docker-compose up -d
sudo systemctl restart haproxy

# 2. 证书验证
md5sum /data/cert/harbor.mysen.top.pem  # 传输后验证文件完整性，两个节点运行对比
openssl verify -CAfile ca.crt harbor.mysen.top.crt  # 验证证书链完整性,OK正常
openssl x509 -in /etc/harbor/ssl/harbor.crt -noout -text | grep -A1 "Subject Alternative Name"  # 验证SAN扩展是否包含所有IP和域名
curl https://10.31.3.242  # 在harbor1上运行，查看返回是否为html标签
curl https://10.31.3.241  # 在harbor2上运行，查看返回是否为html标签
openssl s_client -connect 10.31.3.242:443 -servername harbor.mysen.top | openssl x509 -noout -text  # 在harbor1上运行与下一条命令对比，应该完全一致
openssl s_client -connect 10.31.3.241:443 -servername harbor.mysen.top | openssl x509 -noout -text  # 在harbor2上运行与上一条命令对比，应该完全一致
openssl s_client -connect 10.31.3.250:443 -servername harbor.mysen.top | openssl x509 -noout -text  # 检查HAProxy是否与harbor1/2一致

三、使用手册

1. 镜像同步

还存在复制问题没有解决？？
其他仓库推送到harbor提示证书问题？？

# 在harbor1的web界面操作：
1.1 在局域网内其他电脑上访问harbor1仓库管理地址https://10.31.3.241（默认账户密码为：admin/Harbor12345）
1.2 点击系统管理的子菜单→仓库管理→新建目标
目标名：harbor2
目标url：https://10.31.3.242
不用输入访问ID和密码，点击测试连接，确定。
1.3 点击系统管理的子菜单→复制管理→新建规则
名称：harbor2
目标仓库：harbor2-https://10.31.3.242
触发模式：事件驱动
勾选删除本地资源时同时也删除远程的资源
其他默认，也可根据需要修改。

# 在harbor2的web界面操作：
1.1 在局域网内其他电脑上访问harbor2仓库管理地址https://10.31.3.242（默认账户密码为：admin/Harbor12345）
1.2 点击系统管理的子菜单→仓库管理→新建目标
目标名：harbor1
目标url：https://10.31.3.241
不用输入访问ID和密码，点击测试连接，确定。
1.3 点击系统管理的子菜单→复制管理→新建规则
名称：harbor1
目标仓库：harbor2-https://10.31.3.241
触发模式：事件驱动，勾选删除本地资源时同时也删除远程的资源
其他默认，也可根据需要修改。

2. 镜像拉取

# 1. 获取Calico 3.30.2官方镜像，typha为大规模集群需要
nerdctl pull calico/cni:v3.30.2
nerdctl pull calico/node:v3.30.2
nerdctl pull calico/kube-controllers:v3.30.2
nerdctl pull calico/pod2daemon-flexvol:v3.30.2
nerdctl pull calico/typha:v3.30.2

3. 镜像推送

# 2. 重新标记并推送至Harbor（根据实际主机名修改）
nerdctl tag calico/cni:v3.30.2 harbor.mysen.pro/library/calico-cni:v3.30.2
nerdctl tag calico/node:v3.30.2 harbor.mysen.pro/library/calico-node:v3.30.2
nerdctl tag calico/kube-controllers:v3.30.2 harbor.mysen.pro/library/calico-kube-controllers:v3.30.2
nerdctl tag calico/pod2daemon-flexvol:v3.30.2 harbor.mysen.pro/library/calico-pod2daemon-flexvol:v3.30.2
nerdctl tag calico/typha:v3.30.2 harbor.mysen.pro/library/calico-typha:v3.30.2

# 3. 推送镜像(需先登录Harbor)
nerdctl login harbor.mysen.pro
nerdctl push harbor.mysen.pro/library/calico-cni:v3.30.2
nerdctl push harbor.mysen.pro/library/calico-node:v3.30.2
nerdctl push harbor.mysen.pro/library/calico-kube-controllers:v3.30.2
nerdctl push harbor.mysen.pro/library/calico-pod2daemon-flexvol:v3.30.2
nerdctl push harbor.mysen.pro/library/calico-typha:v3.30.2

Ubuntu 24.04搭建高可用Harbor私有镜像仓库

一、确认配置

二、配置环境

1. 基础环境配置

2. ‌配置containerd‌

3. 生成证书

4. 部署Harbor

5. 负载均衡

6. 部署验证

三、使用手册

1. 镜像同步

2. 镜像拉取

3. 镜像推送

4. 配置k8s拉取私有镜像

四、其他事项

1. 证书续期

2. 安全加固

3. 监控维护

Ubuntu 24.04搭建高可用Harbor私有镜像仓库

一、 确认配置

二、 配置环境

1. 基础环境配置

2. ‌配置containerd‌

3. 生成证书

4. 部署Harbor

5. 负载均衡

6. 部署验证

三、 使用手册

1. 镜像同步

2. 镜像拉取

3. 镜像推送

4. 配置k8s拉取私有镜像

四、 其他事项

1. 证书续期

2. 安全加固

3. 监控维护

一、确认配置

二、配置环境

三、使用手册

四、其他事项