PAE with FreeBSD 4.9-RC - BBS

在下有幸接管一個不算小的 bbs站.
因為我們的系統要使用掉相當多的記憶體,
在不開 PAE時只能抓到 4GB已經不夠使用.
所以就開 PAE來試試看 :p

系統配備如下:
Intel Pentium III Xeon 700 x 2
Transcend 512MB SDRAM ECC Registered x 12 (6GB)
(onboard)AMD PCnet/PCI 10/100BaseTX (pcn0)
(詳細的 dmesg 我貼在最後面)
加上FreeBSD 4.9-RC

FreeBSD 4.9-RC 對 PAE似乎沒有很詳細的說明 (或是我沒有找到?)
不若 5.1R 有 /usr/src/sys/i386/conf/PAE 告知哪些 device 不支援的,
所以還有點麻煩 :(
由 5.1R 的 PAE中得知 pcn目前似乎不支援,
故又加了一張 intel 82559.

基本上只要在 kernel conf中加入 options PAE重新 compile kernel 即可,
唯所有的 module 應該直接 link 進 kernel 中:
Since KLD modules are not compiled with the same options headers
that the kernel is compiled with, they must not be loaded into a
kernel compiled with the PAE option.

若一開機的時候看到類似這樣的訊息:
real memory = 6442450944 (6291456K bytes)
kmem_suballoc: bad status return of 3.
panic: kmem_suballoc
mp_lock = 0000000c; cpuid = 0; lapic.id = 03000000
Uptime: 0s
表示 KVA不夠大, 個人目前是使用
options KVA_PAGES=512
重新 compile後即可正常啟動.


比較奇怪的是一直碰到
fxp0: DMA timeout
fxp0: DMA timeout
fxp0: DMA timeout
fxp0: device timeout
我試了 82559 以及 82558B都碰到同樣的訊息,
網路完全沒有辦法動,
但是將 options PAE拿掉後就沒有任何問題.
所以看起來應該是和 PAE不相容(?)
真是相當怪 :(

後來將 fxp拿掉, 改用 onboard的 pcn, 竟然可以動 @_@
(因為在 5.1R 的 PAE裡面有寫 nodevice pcn @_@)


另外, 在有使用 PAE的時候, 速度感覺比較慢.
例如說進 single user mode 時,
以往 mount /usr 按下 enter後立刻就好了,
而使用 PAE後, 卻要等個一兩秒鐘才會跳出來.
或者是有些程式跑起來 (ex: top, bbsd)
大部份的時間十分順,
卻在某些時候 (還沒有抓到 pattern)
會突然卡一下, 可以卡個一兩秒到五六秒, 所以相當明顯.
可是系統 load 並沒有很大 (cpu idle > 70%)
相當怪 :(

目前只有這樣子, 等到待會兒系統 load 大起來的時候,
也許會有一些特別的東西出現 :p

我們的系統跑在 telnet://ptt.cc
一些相關的統計在 http://mrtg.ptt.cc
我個人的聯絡方式是 in2 _AT_ in2home.org
若您有任何問題或建議歡迎不吝嗇指教 ^_^

---
對了, 我的 ipfw 沒有辦法用, 一用就出現:
ipfw: getsockopt(IP_FW_ADD): Invalid argument
ipfw: getsockopt(IP_FW_ADD): Invalid argument
ipfw: getsockopt(IP_FW_ADD): Invalid argument
有沒有人知道是為什麼呢?
似乎我 kernel config有哪裡設錯了 :(

kernel config --------------------------------------------------------------
machine i386
cpu I686_CPU
ident ptt
maxusers 512

options INET #InterNETworking
options FFS #Berkeley Fast Filesystem
options FFS_ROOT #FFS usable as root device [keep this!]
options SOFTUPDATES #Enable FFS soft updates support
options UFS_DIRHASH #Improve performance on big directories
options NFS #Network Filesystem
options PROCFS #Process filesystem
options COMPAT_43 #Compatible with BSD 4.3 [KEEP THIS!]
options SCSI_DELAY=2000 #Delay (in ms) before probing SCSI
options SYSVSHM #SYSV-style shared memory
options SYSVMSG #SYSV-style message queues
options SYSVSEM #SYSV-style semaphores
options P1003_1B #Posix P1003_1B real-time extensions
options _KPOSIX_PRIORITY_SCHEDULING
options ICMP_BANDLIM #Rate limit bad replies
options KBD_INSTALL_CDEV # install a CDEV entry in /dev
#options AHC_REG_PRETTY_PRINT # Print register bitfields in debug
# output. Adds ~128k to driver.
#options AHD_REG_PRETTY_PRINT # Print register bitfields in debug
# output. Adds ~215k to driver.

# To make an SMP kernel, the next two are needed
options SMP # Symmetric MultiProcessor Kernel
options APIC_IO # Symmetric (APIC) I/O

# To support HyperThreading, HTT is needed in addition to SMP and APIC_IO
#options HTT # HyperThreading Technology

device isa
device eisa
device pci

# SCSI Controllers
device ahc # AHA2940 and onboard AIC7xxx devices

# SCSI peripherals
device scbus # SCSI bus (required)
device da # Direct Access (disks)
device pass # Passthrough device (direct SCSI access)

# atkbdc0 controls both the keyboard and the PS/2 mouse
device atkbdc0 at isa? port IO_KBD
device atkbd0 at atkbdc? irq 1 flags 0x1
device psm0 at atkbdc? irq 12

device vga0 at isa?

# syscons is the default console driver, resembling an SCO console
device sc0 at isa? flags 0x100

# Enable this and PCVT_FREEBSD for pcvt vt220 compatible console driver
#device vt0 at isa?
#options XSERVER # support for X server on a vt console
#options FAT_CURSOR # start with block cursor
# If you have a ThinkPAD, uncomment this along with the rest of the PCVT lines
#options PCVT_SCANSET=2 # IBM keyboards are non-std

# Floating point support - do not disable.
device npx0 at nexus? port IO_NPX irq 13

# Serial (COM) ports
device sio0 at isa? port IO_COM1 flags 0x10 irq 4
device sio1 at isa? port IO_COM2 irq 3
device sio2 at isa? disable port IO_COM3 irq 5
device sio3 at isa? disable port IO_COM4 irq 9

# PCI Ethernet NICs that use the common MII bus controller code.
# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
device miibus # MII bus support
#device fxp # Intel EtherExpress PRO/100B (82557, 82558)
device pcn # AMD Am79C97x PCI 10/100 NICs

# Pseudo devices - the number indicates how many units to allocate.
pseudo-device loop # Network loopback
pseudo-device ether # Ethernet support
pseudo-device pty # Pseudo-ttys (telnet etc)

# The `bpf' pseudo-device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
pseudo-device bpf #Berkeley packet filter

options IPFIREWALL
options IPFIREWALL_DEFAULT_TO_ACCEPT
options NMBCLUSTERS=16000
options AUTO_EOI_1
options CPU_ENABLE_SSE
#options KVA_PAGES=288
options KVA_PAGES=512
options CONSPEED=115200
options COMPAT_LINUX
options PAE

dmesg ----------------------------------------------------------------------
Copyright (c) 1992-2003 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 4.9-RC #0: Tue Oct 7 08:02:29 CST 2003
[email protected]:/usr/obj/usr/src/sys/ptt
Timecounter "i8254" frequency 1193182 Hz
CPU: Intel Pentium III Xeon (701.73-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0x6a1 Stepping = 1
Features=0x387fbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,C\
MOV,PAT,PSE36,PN,MMX,FXSR,SSE>
real memory = 6442450944 (6291456K bytes)
avail memory = 5868003328 (5730472K bytes)
APIC_IO: MP table broken: 8259->APIC entry missing!
Programming 16 pins in IOAPIC #0
IOAPIC #0 intpin 2 -> irq 0
Programming 16 pins in IOAPIC #1
FreeBSD/SMP: Multiprocessor motherboard: 2 CPUs
cpu0 (BSP): apic id: 3, version: 0x00040011, at 0xfee00000
cpu1 (AP): apic id: 0, version: 0x00040011, at 0xfee00000
io0 (APIC): apic id: 14, version: 0x000f0011, at 0xfec00000
io1 (APIC): apic id: 13, version: 0x000f0011, at 0xfec01000
Preloaded elf kernel "kernel" at 0xc0310000.
Pentium Pro MTRR support enabled
npx0: <math processor> on motherboard
npx0: INT 16 interface
pcib2: <ServerWorks host to PCI bridge> on motherboard
IOAPIC #1 intpin 1 -> irq 2
IOAPIC #1 intpin 2 -> irq 9
pci2: <PCI bus> on pcib2
ahc0: <Adaptec aic7899 Ultra160 SCSI adapter> port 0x4000-0x40ff mem
0xefbff000-0xefbfffff irq 2 at device 1.0 on pci2
aic7899: Ultra160 Wide Channel A, SCSI Id=7, 32/253 SCBs
ahc1: <Adaptec aic7899 Ultra160 SCSI adapter> port 0x4100-0x41ff mem
0xefbfe000-0xefbfefff irq 9 at device 1.1 on pci2
aic7899: Ultra160 Wide Channel B, SCSI Id=7, 32/253 SCBs
pcib0: <ServerWorks host to PCI bridge> on motherboard
IOAPIC #1 intpin 0 -> irq 10
IOAPIC #1 intpin 3 -> irq 11
pci0: <PCI bus> on pcib0
pcn0: <AMD PCnet/PCI 10/100BaseTX> port 0x2200-0x221f mem
0xfebffc00-0xfebffc1f irq 10 at device 5.0 on pci0
pcn0: Ethernet address: 00:06:29:f6:11:da
miibus0: <MII bus> on pcn0
ukphy0: <Generic IEEE 802.3u media interface> on miibus0
ukphy0: 10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
pci0: <S3 Savage 4 graphics accelerator> at 6.0
isab0: <ServerWorks IB6566 PCI to ISA bridge> at device 15.0 on pci0
isa0: <ISA bus> on isab0
pci0: <Unknown PCI ATA controller> at 15.1
pci0: <OHCI USB controller> at 15.2 irq 11
pcib3: <ServerWorks host to PCI bridge> on motherboard
pci3: <PCI bus> on pcib3
pcib5: <ServerWorks host to PCI bridge> on motherboard
pci5: <PCI bus> on pcib5
orm0: <Option ROM> at iomem 0xc0000-0xc9fff on isa0
pmtimer0 on isa0
atkbdc0: <Keyboard controller (i8042)> at port 0x60,0x64 on isa0
atkbd0: <AT Keyboard> flags 0x1 irq 1 on atkbdc0
device_probe_and_attach: atkbd0 attach returned 6
vga0: <Generic ISA VGA> at port 0x3c0-0x3df iomem 0xa0000-0xbffff on isa0
sc0: <System console> at flags 0x100 on isa0
sc0: VGA <16 virtual consoles, flags=0x100>
sio0 at port 0x3f8-0x3ff irq 4 flags 0x10 on isa0
sio0: type 16550A, console
sio1 at port 0x2f8-0x2ff irq 3 on isa0
sio1: type 16550A
APIC_IO: Testing 8254 interrupt delivery
APIC_IO: Broken MP table detected: 8254 is not connected to IOAPIC #0 intpin 2
APIC_IO: routing 8254 via 8259 and IOAPIC #0 intpin 0
IP packet filtering initialized, divert disabled, rule-based forwarding
enabled, default to accept, logging disabled
Waiting 2 seconds for SCSI devices to settle
SMP: AP CPU #1 Launched!
pass3 at ahc0 bus 0 target 8 lun 0
pass3: <IBM YGLv3 S2 0> Fixed Processor SCSI-2 device
pass3: 3.300MB/s transfers
da4 at ahc1 bus 0 target 4 lun 0
da4: <ATA WDC WD800JB- 17.0> Fixed Direct Access SCSI-3 device
da4: 160.000MB/s transfers (80.000MHz, offset 14, 16bit)
da4: 76319MB (156301488 512 byte sectors: 255H 63S/T 9729C)
da0 at ahc0 bus 0 target 0 lun 0
da0: <IBM DDYS-T36950M S80D> Fixed Direct Access SCSI-3 device
da0: 160.000MB/s transfers (80.000MHz, offset 63, 16bit), Tagged Queueing
Enabled
da0: 35003MB (71687340 512 byte sectors: 255H 63S/T 4462C)
da5 at ahc1 bus 0 target 5 lun 0
da5: <ATA WDC WD800JB- 17.0> Fixed Direct Access SCSI-3 device
da5: 160.000MB/s transfers (80.000MHz, offset 14, 16bit)
da5: 76319MB (156301488 512 byte sectors: 255H 63S/T 9729C)
da6 at ahc1 bus 0 target 6 lun 0
da6: <ATA WDC WD800JB- 17.0> Fixed Direct Access SCSI-3 device
da6: 160.000MB/s transfers (80.000MHz, offset 14, 16bit)
da6: 76319MB (156301488 512 byte sectors: 255H 63S/T 9729C)
da3 at ahc1 bus 0 target 2 lun 0
da3: <FUJITSU MAN3367MP 0108> Fixed Direct Access SCSI-3 device
da3: 160.000MB/s transfers (80.000MHz, offset 127, 16bit), Tagged Queueing
Enabled
da3: 35044MB (71771688 512 byte sectors: 255H 63S/T 4467C)
da1 at ahc0 bus 0 target 1 lun 0
da1: <IBM DDYS-T36950M S80D> Fixed Direct Access SCSI-3 device
da1: 160.000MB/s transfers (80.000MHz, offset 63, 16bit), Tagged Queueing
Enabled
da1: 35003MB (71687340 512 byte sectors: 255H 63S/T 4462C)
da8 at ahc1 bus 0 target 10 lun 0
da8: <ATA WDC WD800JB- 77.0> Fixed Direct Access SCSI-3 device
da8: 160.000MB/s transfers (80.000MHz, offset 14, 16bit)
da8: 76319MB (156301488 512 byte sectors: 255H 63S/T 9729C)
da9 at ahc1 bus 0 target 11 lun 0
da9: <ATA WDC WD800JB- 77.0> Fixed Direct Access SCSI-3 device
da9: 160.000MB/s transfers (80.000MHz, offset 14, 16bit)
da9: 76319MB (156301488 512 byte sectors: 255H 63S/T 9729C)
da2 at ahc0 bus 0 target 2 lun 0
da2: <IBM DDYS-T36950M S80D> Fixed Direct Access SCSI-3 device
da2: 160.000MB/s transfers (80.000MHz, offset 63, 16bit), Tagged Queueing
Enabled
da2: 35003MB (71687340 512 byte sectors: 255H 63S/T 4462C)
da7 at ahc1 bus 0 target 9 lun 0
da7: <ATA WDC WD800JB- 17.0> Fixed Direct Access SCSI-3 device
da7: 160.000MB/s transfers (80.000MHz, offset 14, 16bit)
da7: 76319MB (156301488 512 byte sectors: 255H 63S/T 9729C)
da10 at ahc1 bus 0 target 12 lun 0
da10: <SEAGATE SX118202LS B804> Fixed Direct Access SCSI-2 device
da10: 80.000MB/s transfers (40.000MHz, offset 15, 16bit), Tagged Queueing
Enabled
da10: 17366MB (35566480 512 byte sectors: 255H 63S/T 2213C)
Mounting root from ufs:/dev/da0s1a

--