0%

上传镜像文件

1
2
3
root@HomeLab:~/files# ls -l
total 1180164
-rw-r--r-- 1 root root 1208483840 Feb 25 22:11 openwrt-x86-64-generic-ext4-combined.img

挂载镜像文件至虚拟机

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# 挂载
root@HomeLab:~/files# qm importdisk 100 /root/files/openwrt-x86-64-generic-ext4-combined.img local-lvm
importing disk '/root/files/openwrt-x86-64-generic-ext4-combined.img' to VM 100 ...
Rounding up size to full physical extent <1.13 GiB
Logical volume "vm-100-disk-0" created.
transferred 0.0 B of 1.1 GiB (0.00%)
transferred 12.0 MiB of 1.1 GiB (1.04%)
transferred 24.0 MiB of 1.1 GiB (2.08%)
transferred 36.0 MiB of 1.1 GiB (3.12%)
transferred 47.9 MiB of 1.1 GiB (4.16%)
transferred 60.0 MiB of 1.1 GiB (5.21%)
···
transferred 1.1 GiB of 1.1 GiB (99.96%)
transferred 1.1 GiB of 1.1 GiB (100.00%)
transferred 1.1 GiB of 1.1 GiB (100.00%)
Successfully imported disk as 'unused0:local-lvm:vm-100-disk-0'

修改 IP

1
vim /etc/config/network

PVE 设置硬件直通

参考文档:

建议参考官方文档。

写在前面

BIOS 开启以下设置

  • Launch CSM: Enabled
  • Boot option filter: UEFI and legacy
  • Launch PXE OpROM policy: UEFI Only
  • Launch Storage OpROM policy: UEFI Only
  • Launch Video OpROM policy: Legacy Only
  • x2apic:Enabled
  • VT-x: Enabled
  • VT-d: Enabled

开启 IOMMU

修改 /etc/default/grub 文件

1
root@HomeLab ~$ vim /etc/default/grub
1
2
3
# GRUB_CMDLINE_LINUX_DEFAULT 修改为下面这样
# Intel CPU
GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt"

更新 grub

1
2
3
4
5
6
7
8
9
10
11
12
root@HomeLab:~# vim /etc/default/grub
root@HomeLab:~# update-grub
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-5.15.85-1-pve
Found initrd image: /boot/initrd.img-5.15.85-1-pve
Found memtest86+ image: /boot/memtest86+.bin
Found memtest86+ multiboot image: /boot/memtest86+_multiboot.bin
Warning: os-prober will not be executed to detect other bootable partitions.
Systems on them will not be added to the GRUB boot configuration.
Check GRUB_DISABLE_OS_PROBER documentation entry.
Adding boot menu entry for UEFI Firmware Settings ...
done

更新 systemd-boot

1
2
3
4
5
root@HomeLab:~# pve-efiboot-tool refresh
Running hook script 'proxmox-auto-removal'..
Running hook script 'zz-proxmox-boot'..
Re-executing '/etc/kernel/postinst.d/zz-proxmox-boot' in new private mount namespace..
No /etc/kernel/proxmox-boot-uuids found, skipping ESP sync.

重启 PVE

1
root@HomeLab ~$ reboot

验证是否开启 IOMMU

出现 DMAR: IOMMU enabled 说明开启成功。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
root@HomeLab ~$ dmesg | grep -e DMAR -e IOMMU
[ 0.011885] ACPI: DMAR 0x0000000076CDF668 0000D4 (v01 ALASKA A M I 00000001 INTL 20091013)
[ 0.011914] ACPI: Reserving DMAR table memory at [mem 0x76cdf668-0x76cdf73b]
[ 0.135283] DMAR: IOMMU enabled
[ 0.372218] DMAR: Host address width 46
[ 0.372220] DMAR: DRHD base: 0x000000fbffd000 flags: 0x0
[ 0.372225] DMAR: dmar0: reg_base_addr fbffd000 ver 1:0 cap d2008c10ef0466 ecap f0205b
[ 0.372228] DMAR: DRHD base: 0x000000fbffc000 flags: 0x1
[ 0.372232] DMAR: dmar1: reg_base_addr fbffc000 ver 1:0 cap d2078c106f0466 ecap f020df
[ 0.372234] DMAR: RMRR base: 0x000000773e0000 end: 0x000000773f0fff
[ 0.372236] DMAR: ATSR flags: 0x0
[ 0.372237] DMAR: RHSA base: 0x000000fbffc000 proximity domain: 0x0
[ 0.372240] DMAR-IR: IOAPIC id 1 under DRHD base 0xfbffc000 IOMMU 1
[ 0.372242] DMAR-IR: IOAPIC id 2 under DRHD base 0xfbffc000 IOMMU 1
[ 0.372243] DMAR-IR: HPET id 0 under DRHD base 0xfbffc000
[ 0.372244] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping.
[ 0.372875] DMAR-IR: Enabled IRQ remapping in x2apic mode
[ 0.840583] DMAR: No SATC found
[ 0.840585] DMAR: IOMMU feature sc_support inconsistent
[ 0.840587] DMAR: IOMMU feature dev_iotlb_support inconsistent
[ 0.840588] DMAR: dmar0: Using Queued invalidation
[ 0.840592] DMAR: dmar1: Using Queued invalidation
[ 0.843805] DMAR: Intel(R) Virtualization Technology for Directed I/O

添加必要的模块

1
root@HomeLab ~$ vim /etc/modules

添加下面内容:

1
2
3
4
vfio
vfio_pci
vfio_virqfd
vfio_iommu_type1

IOMMU Interrupt Remapping

1
2
3
root@HomeLab ~$ dmesg | grep 'remapping'
[ 0.372244] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping.
[ 0.372875] DMAR-IR: Enabled IRQ remapping in x2apic mode

出现下面其中一条内容没说明支持 IOMMU Interrupt Remapping

  • “AMD-Vi: Interrupt remapping enabled”
  • “DMAR-IR: Enabled IRQ remapping in x2apic mode” (‘x2apic’ can be different on old CPUs, but should still work)

如果没有,则需要执行:

1
echo "options vfio_iommu_type1 allow_unsafe_interrupts=1" > /etc/modprobe.d/iommu_unsafe_interrupts.conf

验证 IOMMU 隔离

执行命令后出现下面的内容,说明支持 IOMMU 分组,可以直通:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
root@HomeLab:~# find /sys/kernel/iommu_groups/ -type l
/sys/kernel/iommu_groups/17/devices/0000:00:05.1
/sys/kernel/iommu_groups/7/devices/0000:ff:14.7
/sys/kernel/iommu_groups/7/devices/0000:ff:14.5
/sys/kernel/iommu_groups/7/devices/0000:ff:14.3
/sys/kernel/iommu_groups/7/devices/0000:ff:14.1
/sys/kernel/iommu_groups/7/devices/0000:ff:14.6
/sys/kernel/iommu_groups/7/devices/0000:ff:14.4
/sys/kernel/iommu_groups/7/devices/0000:ff:14.2
/sys/kernel/iommu_groups/7/devices/0000:ff:14.0
/sys/kernel/iommu_groups/25/devices/0000:00:1d.0
/sys/kernel/iommu_groups/15/devices/0000:00:03.0
/sys/kernel/iommu_groups/33/devices/0000:06:00.0
/sys/kernel/iommu_groups/5/devices/0000:ff:12.4
/sys/kernel/iommu_groups/5/devices/0000:ff:12.0
/sys/kernel/iommu_groups/5/devices/0000:ff:12.5
/sys/kernel/iommu_groups/5/devices/0000:ff:12.1
/sys/kernel/iommu_groups/23/devices/0000:00:1c.0
/sys/kernel/iommu_groups/13/devices/0000:00:01.0
/sys/kernel/iommu_groups/31/devices/0000:03:00.0
/sys/kernel/iommu_groups/3/devices/0000:ff:0f.6
/sys/kernel/iommu_groups/3/devices/0000:ff:0f.4
/sys/kernel/iommu_groups/3/devices/0000:ff:0f.2
/sys/kernel/iommu_groups/3/devices/0000:ff:0f.0
/sys/kernel/iommu_groups/3/devices/0000:ff:0f.5
/sys/kernel/iommu_groups/3/devices/0000:ff:0f.3
/sys/kernel/iommu_groups/3/devices/0000:ff:0f.1
/sys/kernel/iommu_groups/21/devices/0000:00:1a.0
/sys/kernel/iommu_groups/11/devices/0000:ff:1f.0
/sys/kernel/iommu_groups/11/devices/0000:ff:1f.2
/sys/kernel/iommu_groups/1/devices/0000:ff:0c.6
/sys/kernel/iommu_groups/1/devices/0000:ff:0c.4
/sys/kernel/iommu_groups/1/devices/0000:ff:0c.2
/sys/kernel/iommu_groups/1/devices/0000:ff:0c.0
/sys/kernel/iommu_groups/1/devices/0000:ff:0c.7
/sys/kernel/iommu_groups/1/devices/0000:ff:0c.5
/sys/kernel/iommu_groups/1/devices/0000:ff:0c.3
/sys/kernel/iommu_groups/1/devices/0000:ff:0c.1
/sys/kernel/iommu_groups/28/devices/0000:01:00.1
/sys/kernel/iommu_groups/18/devices/0000:00:05.2
/sys/kernel/iommu_groups/8/devices/0000:ff:16.2
/sys/kernel/iommu_groups/8/devices/0000:ff:16.0
/sys/kernel/iommu_groups/8/devices/0000:ff:16.7
/sys/kernel/iommu_groups/8/devices/0000:ff:16.3
/sys/kernel/iommu_groups/8/devices/0000:ff:16.1
/sys/kernel/iommu_groups/8/devices/0000:ff:16.6
/sys/kernel/iommu_groups/26/devices/0000:00:1f.0
/sys/kernel/iommu_groups/26/devices/0000:00:1f.3
/sys/kernel/iommu_groups/16/devices/0000:00:05.0
/sys/kernel/iommu_groups/6/devices/0000:ff:13.2
/sys/kernel/iommu_groups/6/devices/0000:ff:13.0
/sys/kernel/iommu_groups/6/devices/0000:ff:13.7
/sys/kernel/iommu_groups/6/devices/0000:ff:13.3
/sys/kernel/iommu_groups/6/devices/0000:ff:13.1
/sys/kernel/iommu_groups/6/devices/0000:ff:13.6
/sys/kernel/iommu_groups/24/devices/0000:00:1c.3
/sys/kernel/iommu_groups/14/devices/0000:00:01.1
/sys/kernel/iommu_groups/32/devices/0000:04:00.1
/sys/kernel/iommu_groups/32/devices/0000:04:00.0
/sys/kernel/iommu_groups/4/devices/0000:ff:10.0
/sys/kernel/iommu_groups/4/devices/0000:ff:10.7
/sys/kernel/iommu_groups/4/devices/0000:ff:10.5
/sys/kernel/iommu_groups/4/devices/0000:ff:10.1
/sys/kernel/iommu_groups/4/devices/0000:ff:10.6
/sys/kernel/iommu_groups/22/devices/0000:00:1b.0
/sys/kernel/iommu_groups/12/devices/0000:00:00.0
/sys/kernel/iommu_groups/30/devices/0000:01:00.3
/sys/kernel/iommu_groups/2/devices/0000:ff:0d.7
/sys/kernel/iommu_groups/2/devices/0000:ff:0d.5
/sys/kernel/iommu_groups/2/devices/0000:ff:0d.3
/sys/kernel/iommu_groups/2/devices/0000:ff:0d.1
/sys/kernel/iommu_groups/2/devices/0000:ff:0d.6
/sys/kernel/iommu_groups/2/devices/0000:ff:0d.4
/sys/kernel/iommu_groups/2/devices/0000:ff:0d.2
/sys/kernel/iommu_groups/2/devices/0000:ff:0d.0
/sys/kernel/iommu_groups/20/devices/0000:00:14.0
/sys/kernel/iommu_groups/10/devices/0000:ff:1e.4
/sys/kernel/iommu_groups/10/devices/0000:ff:1e.2
/sys/kernel/iommu_groups/10/devices/0000:ff:1e.0
/sys/kernel/iommu_groups/10/devices/0000:ff:1e.3
/sys/kernel/iommu_groups/10/devices/0000:ff:1e.1
/sys/kernel/iommu_groups/29/devices/0000:01:00.2
/sys/kernel/iommu_groups/0/devices/0000:ff:0b.1
/sys/kernel/iommu_groups/0/devices/0000:ff:0b.2
/sys/kernel/iommu_groups/0/devices/0000:ff:0b.0
/sys/kernel/iommu_groups/19/devices/0000:00:05.4
/sys/kernel/iommu_groups/9/devices/0000:ff:17.7
/sys/kernel/iommu_groups/9/devices/0000:ff:17.5
/sys/kernel/iommu_groups/9/devices/0000:ff:17.3
/sys/kernel/iommu_groups/9/devices/0000:ff:17.1
/sys/kernel/iommu_groups/9/devices/0000:ff:17.6
/sys/kernel/iommu_groups/9/devices/0000:ff:17.4
/sys/kernel/iommu_groups/9/devices/0000:ff:17.2
/sys/kernel/iommu_groups/9/devices/0000:ff:17.0
/sys/kernel/iommu_groups/27/devices/0000:01:00.0

黑群晖

硬件直通

在 PVE 中添加 PCIE 设备时提示: No IOMMU detected, please activate it.See Documentation for further information.,需要开启 iommu。

阅读全文 »

修改基础系统(Debian)的源文件 /etc/apt/sources.list 和 Proxmox 的源文件

修改基础系统(Debian)的源文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# 备份
root@HomeLab:~# cp /etc/apt/sources.list /etc/apt/sources.list_backup
root@HomeLab:~# cp /etc/apt/sources.list.d/pve-enterprise.list /etc/apt/sources.list.d/pve-enterprise.list_backup
# 替换成中科大换
root@HomeLab:~# sed -i 's|^deb http://ftp.debian.org|deb https://mirrors.ustc.edu.cn|g' /etc/apt/sources.list
root@HomeLab:~# sed -i 's|^deb http://security.debian.org|deb https://mirrors.ustc.edu.cn/debian-security|g' /etc/apt/sources.list
# 查看替换结果
root@HomeLab:~# cat /etc/apt/sources.list
deb https://mirrors.ustc.edu.cn/debian bullseye main contrib

deb https://mirrors.ustc.edu.cn/debian bullseye-updates main contrib

# security updates
deb https://mirrors.ustc.edu.cn/debian-security bullseye-security main contrib
阅读全文 »

默认日志格式

logback

1
%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n

SpringBoot

1
%d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${PID:-} [%15.15t] %-40.40logger{39} : %m%n

logback 设置

application.yml

1
2
logging:
config: classpath:logback-${spring.profiles.active}.xml

STDOUT

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">

<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %highlight(%5p) %yellow(${PID:-}) --- [%15.15t] %cyan(%-40.40logger{39}) : %m%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>

<root level="INFO">
<appender-ref ref="STDOUT"/>
</root>

</configuration>

FILE

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">

<property name="LOG_FILE_PATH" value="./logs"/>

<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${LOG_FILE_PATH}/anthology.%d{yyyy-MM-dd}.log</FileNamePattern>
<MaxHistory>30</MaxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %highlight(%5p) %boldGreen(${PID:-}) --- [%15.15t] %cyan(%-40.40logger{39}) : %m%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>

<root level="INFO">
<appender-ref ref="FILE"/>
</root>

</configuration>