From 213a1276c76440c2509354c37dab12fd3a38ef62 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 31 May 2015 17:24:47 +0200 Subject: [PATCH 001/257] amd64: Save a copy of GENERIC as ELECTRO_BLOAT Obtained from: ElectroBSD --- sys/amd64/conf/ELECTRO_BLOAT | 365 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 sys/amd64/conf/ELECTRO_BLOAT diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT new file mode 100644 index 000000000000..3d2cc39c6353 --- /dev/null +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -0,0 +1,365 @@ +# +# GENERIC -- Generic kernel configuration file for FreeBSD/amd64 +# +# For more information on this file, please read the config(5) manual page, +# and/or the handbook section on Kernel Configuration Files: +# +# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html +# +# The handbook is also available locally in /usr/share/doc/handbook +# if you've installed the doc distribution, otherwise always see the +# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the +# latest information. +# +# An exhaustive list of options and more detailed explanations of the +# device lines is also present in the ../../conf/NOTES and NOTES files. +# If you are in doubt as to the purpose or necessity of a line, check first +# in NOTES. +# +# $FreeBSD$ + +cpu HAMMER +ident GENERIC + +makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols +makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support + +options SCHED_ULE # ULE scheduler +options PREEMPTION # Enable kernel thread preemption +options INET # InterNETworking +options INET6 # IPv6 communications protocols +options TCP_OFFLOAD # TCP offload +options SCTP # Stream Control Transmission Protocol +options FFS # Berkeley Fast Filesystem +options SOFTUPDATES # Enable FFS soft updates support +options UFS_ACL # Support for access control lists +options UFS_DIRHASH # Improve performance on big directories +options UFS_GJOURNAL # Enable gjournal-based UFS journaling +options QUOTA # Enable disk quotas for UFS +options MD_ROOT # MD is a potential root device +options NFSCL # Network Filesystem Client +options NFSD # Network Filesystem Server +options NFSLOCKD # Network Lock Manager +options NFS_ROOT # NFS usable as /, requires NFSCL +options MSDOSFS # MSDOS Filesystem +options CD9660 # ISO 9660 Filesystem +options PROCFS # Process filesystem (requires PSEUDOFS) +options PSEUDOFS # Pseudo-filesystem framework +options GEOM_PART_GPT # GUID Partition Tables. +options GEOM_RAID # Soft RAID functionality. +options GEOM_LABEL # Provides labelization +options COMPAT_FREEBSD32 # Compatible with i386 binaries +options COMPAT_FREEBSD4 # Compatible with FreeBSD4 +options COMPAT_FREEBSD5 # Compatible with FreeBSD5 +options COMPAT_FREEBSD6 # Compatible with FreeBSD6 +options COMPAT_FREEBSD7 # Compatible with FreeBSD7 +options COMPAT_FREEBSD9 # Compatible with FreeBSD9 +options COMPAT_FREEBSD10 # Compatible with FreeBSD10 +options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI +options KTRACE # ktrace(1) support +options STACK # stack(9) support +options SYSVSHM # SYSV-style shared memory +options SYSVMSG # SYSV-style message queues +options SYSVSEM # SYSV-style semaphores +options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions +options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed. +options KBD_INSTALL_CDEV # install a CDEV entry in /dev +options HWPMC_HOOKS # Necessary kernel hooks for hwpmc(4) +options AUDIT # Security event auditing +options CAPABILITY_MODE # Capsicum capability mode +options CAPABILITIES # Capsicum capabilities +options MAC # TrustedBSD MAC Framework +options KDTRACE_FRAME # Ensure frames are compiled in +options KDTRACE_HOOKS # Kernel DTrace hooks +options DDB_CTF # Kernel ELF linker loads CTF data +options INCLUDE_CONFIG_FILE # Include this file in kernel +options RACCT # Resource accounting framework +options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default +options RCTL # Resource limits + +# Debugging support. Always need this: +options KDB # Enable kernel debugger support. +options KDB_TRACE # Print a stack trace for a panic. +# For full debugger support use (turn off in stable branch): +options DDB # Support DDB. +options GDB # Support remote GDB. +options DEADLKRES # Enable the deadlock resolver +options INVARIANTS # Enable calls of extra sanity checking +options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS +options WITNESS # Enable checks to detect deadlocks and cycles +options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones + +# Make an SMP-capable kernel by default +options SMP # Symmetric MultiProcessor Kernel + +# CPU frequency control +device cpufreq + +# Bus support. +device acpi +options ACPI_DMAR +device pci +options PCI_IOV # PCI SR-IOV support + +# Floppy drives +device fdc + +# ATA controllers +device ahci # AHCI-compatible SATA controllers +device ata # Legacy ATA/SATA controllers +options ATA_STATIC_ID # Static device numbering +device mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA +device siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA + +# SCSI Controllers +device ahc # AHA2940 and onboard AIC7xxx devices +options AHC_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~128k to driver. +device ahd # AHA39320/29320 and onboard AIC79xx devices +options AHD_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~215k to driver. +device esp # AMD Am53C974 (Tekram DC-390(T)) +device hptiop # Highpoint RocketRaid 3xxx series +device isp # Qlogic family +#device ispfw # Firmware for QLogic HBAs- normally a module +device mpt # LSI-Logic MPT-Fusion +device mps # LSI-Logic MPT-Fusion 2 +device mpr # LSI-Logic MPT-Fusion 3 +#device ncr # NCR/Symbios Logic +device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') +device trm # Tekram DC395U/UW/F DC315U adapters + +device adv # Advansys SCSI adapters +device adw # Advansys wide SCSI adapters +device aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60. +device bt # Buslogic/Mylex MultiMaster SCSI adapters +device isci # Intel C600 SAS controller + +# ATA/SCSI peripherals +device scbus # SCSI bus (required for ATA/SCSI) +device ch # SCSI media changers +device da # Direct Access (disks) +device sa # Sequential Access (tape etc) +device cd # CD +device pass # Passthrough device (direct ATA/SCSI access) +device ses # Enclosure Services (SES and SAF-TE) +#device ctl # CAM Target Layer + +# RAID controllers interfaced to the SCSI subsystem +device amr # AMI MegaRAID +device arcmsr # Areca SATA II RAID +device ciss # Compaq Smart RAID 5* +device dpt # DPT Smartcache III, IV - See NOTES for options +device hptmv # Highpoint RocketRAID 182x +device hptnr # Highpoint DC7280, R750 +device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx +device hpt27xx # Highpoint RocketRAID 27xx +device iir # Intel Integrated RAID +device ips # IBM (Adaptec) ServeRAID +device mly # Mylex AcceleRAID/eXtremeRAID +device twa # 3ware 9000 series PATA/SATA RAID +device tws # LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller + +# RAID controllers +device aac # Adaptec FSA RAID +device aacp # SCSI passthrough for aac (requires CAM) +device aacraid # Adaptec by PMC RAID +device ida # Compaq Smart RAID +device mfi # LSI MegaRAID SAS +device mlx # Mylex DAC960 family +device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s +#XXX pointer/int warnings +#device pst # Promise Supertrak SX6000 +device twe # 3ware ATA RAID + +# NVM Express (NVMe) support +device nvme # base NVMe driver +device nvd # expose NVMe namespaces as disks, depends on nvme + +# atkbdc0 controls both the keyboard and the PS/2 mouse +device atkbdc # AT keyboard controller +device atkbd # AT keyboard +device psm # PS/2 mouse + +device kbdmux # keyboard multiplexer + +device vga # VGA video card driver +options VESA # Add support for VESA BIOS Extensions (VBE) + +device splash # Splash screen and screen saver support + +# syscons is the default console driver, resembling an SCO console +device sc +options SC_PIXEL_MODE # add support for the raster text mode + +# vt is the new video console driver +device vt +device vt_vga +device vt_efifb + +device agp # support several AGP chipsets + +# PCCARD (PCMCIA) support +# PCMCIA and cardbus bridge support +device cbb # cardbus (yenta) bridge +device pccard # PC Card (16-bit) bus +device cardbus # CardBus (32-bit) bus + +# Serial (COM) ports +device uart # Generic UART driver + +# Parallel port +device ppc +device ppbus # Parallel port bus (required) +device lpt # Printer +device ppi # Parallel port interface device +#device vpo # Requires scbus and da + +device puc # Multi I/O cards and multi-channel UARTs + +# PCI Ethernet NICs. +device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE +device de # DEC/Intel DC21x4x (``Tulip'') +device em # Intel PRO/1000 Gigabit Ethernet Family +device igb # Intel PRO/1000 PCIE Server Gigabit Family +device ix # Intel PRO/10GbE PCIE PF Ethernet +device ixv # Intel PRO/10GbE PCIE VF Ethernet +device ixl # Intel XL710 40Gbe PCIE Ethernet +device ixlv # Intel XL710 40Gbe VF PCIE Ethernet +device le # AMD Am7900 LANCE and Am79C9xx PCnet +device ti # Alteon Networks Tigon I/II gigabit Ethernet +device txp # 3Com 3cR990 (``Typhoon'') +device vx # 3Com 3c590, 3c595 (``Vortex'') + +# PCI Ethernet NICs that use the common MII bus controller code. +# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs! +device miibus # MII bus support +device ae # Attansic/Atheros L2 FastEthernet +device age # Attansic/Atheros L1 Gigabit Ethernet +device alc # Atheros AR8131/AR8132 Ethernet +device ale # Atheros AR8121/AR8113/AR8114 Ethernet +device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet +device bfe # Broadcom BCM440x 10/100 Ethernet +device bge # Broadcom BCM570xx Gigabit Ethernet +device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn +device dc # DEC/Intel 21143 and various workalikes +device et # Agere ET1310 10/100/Gigabit Ethernet +device fxp # Intel EtherExpress PRO/100B (82557, 82558) +device gem # Sun GEM/Sun ERI/Apple GMAC +device hme # Sun HME (Happy Meal Ethernet) +device jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet +device lge # Level 1 LXT1001 gigabit Ethernet +device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet +device nfe # nVidia nForce MCP on-board Ethernet +device nge # NatSemi DP83820 gigabit Ethernet +device pcn # AMD Am79C97x PCI 10/100 (precedence over 'le') +device re # RealTek 8139C+/8169/8169S/8110S +device rl # RealTek 8129/8139 +device sf # Adaptec AIC-6915 (``Starfire'') +device sge # Silicon Integrated Systems SiS190/191 +device sis # Silicon Integrated Systems SiS 900/SiS 7016 +device sk # SysKonnect SK-984x & SK-982x gigabit Ethernet +device ste # Sundance ST201 (D-Link DFE-550TX) +device stge # Sundance/Tamarack TC9021 gigabit Ethernet +device tl # Texas Instruments ThunderLAN +device tx # SMC EtherPower II (83c170 ``EPIC'') +device vge # VIA VT612x gigabit Ethernet +device vr # VIA Rhine, Rhine II +device wb # Winbond W89C840F +device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'') + +# Wireless NIC cards +device wlan # 802.11 support +options IEEE80211_DEBUG # enable debug msgs +options IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's +options IEEE80211_SUPPORT_MESH # enable 802.11s draft support +device wlan_wep # 802.11 WEP support +device wlan_ccmp # 802.11 CCMP support +device wlan_tkip # 802.11 TKIP support +device wlan_amrr # AMRR transmit rate control algorithm +device an # Aironet 4500/4800 802.11 wireless NICs. +device ath # Atheros NICs +device ath_pci # Atheros pci/cardbus glue +device ath_hal # pci/cardbus chip support +options AH_SUPPORT_AR5416 # enable AR5416 tx/rx descriptors +options AH_AR5416_INTERRUPT_MITIGATION # AR5416 interrupt mitigation +options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later +device ath_rate_sample # SampleRate tx rate control for ath +#device bwi # Broadcom BCM430x/BCM431x wireless NICs. +#device bwn # Broadcom BCM43xx wireless NICs. +device ipw # Intel 2100 wireless NICs. +device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. +device iwn # Intel 4965/1000/5000/6000 wireless NICs. +device malo # Marvell Libertas wireless NICs. +device mwl # Marvell 88W8363 802.11n wireless NICs. +device ral # Ralink Technology RT2500 wireless NICs. +device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. +device wpi # Intel 3945ABG wireless NICs. + +# Pseudo devices. +device loop # Network loopback +device random # Entropy device +device padlock_rng # VIA Padlock RNG +device rdrand_rng # Intel Bull Mountain RNG +device ether # Ethernet support +device vlan # 802.1Q VLAN support +device tun # Packet tunnel. +device md # Memory "disks" +device gif # IPv6 and IPv4 tunneling +device firmware # firmware assist module + +# The `bpf' device enables the Berkeley Packet Filter. +# Be aware of the administrative consequences of enabling this! +# Note that 'bpf' is required for DHCP. +device bpf # Berkeley packet filter + +# USB support +options USB_DEBUG # enable debug msgs +device uhci # UHCI PCI->USB interface +device ohci # OHCI PCI->USB interface +device ehci # EHCI PCI->USB interface (USB 2.0) +device xhci # XHCI PCI->USB interface (USB 3.0) +device usb # USB Bus (required) +device ukbd # Keyboard +device umass # Disks/Mass storage - Requires scbus and da + +# Sound support +device sound # Generic sound driver (required) +device snd_cmi # CMedia CMI8338/CMI8738 +device snd_csa # Crystal Semiconductor CS461x/428x +device snd_emu10kx # Creative SoundBlaster Live! and Audigy +device snd_es137x # Ensoniq AudioPCI ES137x +device snd_hda # Intel High Definition Audio +device snd_ich # Intel, NVidia and other ICH AC'97 Audio +device snd_via8233 # VIA VT8233x Audio + +# MMC/SD +device mmc # MMC/SD bus +device mmcsd # MMC/SD memory card +device sdhci # Generic PCI SD Host Controller + +# VirtIO support +device virtio # Generic VirtIO bus (required) +device virtio_pci # VirtIO PCI device +device vtnet # VirtIO Ethernet device +device virtio_blk # VirtIO Block device +device virtio_scsi # VirtIO SCSI device +device virtio_balloon # VirtIO Memory Balloon device + +# HyperV drivers and enchancement support +# NOTE: HYPERV depends on hyperv. They must be added or removed together. +options HYPERV # Hyper-V kernel infrastructure +device hyperv # HyperV drivers + +# Xen HVM Guest Optimizations +# NOTE: XENHVM depends on xenpci. They must be added or removed together. +options XENHVM # Xen HVM kernel infrastructure +device xenpci # Xen HVM Hypervisor services driver + +# VMware support +device vmx # VMware VMXNET3 Ethernet + +# Netmap provides direct access to TX/RX rings on supported NICs +device netmap # netmap(4) support + -- 2.11.0 From 112e808aa0e15a3642b113928caa7c959ff8380d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 31 May 2015 17:30:20 +0200 Subject: [PATCH 002/257] ELECTRO_BLOAT: ElectroBSDify the copy Mainly by removing stuff that depends on proprietary software ElectroBSD doesn't ship with or stuff that isn't relevant for ElectroBSD. While at it, import the 'device crypto' line from GENERIC. While we don't need it for IPSEC, having it in the kernel makes upgrading from FreeBSD to ElectroBSD more convenient when using cloudiatr. By default cloudiatr only adds currenty-loaded modules to the bpool, so if crypto.ko is part of the currently running kernel, but the installed kernel requires it as module, the newly-setup system will not boot unless the user adds the module manually. Obtained from: ElectroBSD --- sys/amd64/conf/ELECTRO_BLOAT | 76 +++++++++++++------------------------------- 1 file changed, 22 insertions(+), 54 deletions(-) diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT index 3d2cc39c6353..a61f4e5eee3a 100644 --- a/sys/amd64/conf/ELECTRO_BLOAT +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -1,25 +1,25 @@ -# -# GENERIC -- Generic kernel configuration file for FreeBSD/amd64 -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# -# $FreeBSD$ +# ELECTRO_BLOAT -- Modified copy of the GENERIC kernel configuration file +# Used for the release media. +ident ELECTRO_BLOAT + +# One of ElectroBSD's most important features: +# hacker-movie-compatible colors by default! +options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK) + +# Add HTTP accept filter support. The "performance gains" might +# be dubious, but adding it results in nicer logs for applications +# that use it (because requests are less intangled). +options ACCEPT_FILTER_HTTP + +# Build pf into the kernel. It doesn't hurt and supposedly +# works around various bugs that only affect the module build. +device pf + +############################################################################## +# Everything below comes from GENERIC, but "offending" lines have been removed +############################################################################## cpu HAMMER -ident GENERIC makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support @@ -29,7 +29,6 @@ options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols options TCP_OFFLOAD # TCP offload -options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support options UFS_ACL # Support for access control lists @@ -84,10 +83,6 @@ options KDB_TRACE # Print a stack trace for a panic. options DDB # Support DDB. options GDB # Support remote GDB. options DEADLKRES # Enable the deadlock resolver -options INVARIANTS # Enable calls of extra sanity checking -options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS -options WITNESS # Enable checks to detect deadlocks and cycles -options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # Make an SMP-capable kernel by default @@ -102,9 +97,6 @@ options ACPI_DMAR device pci options PCI_IOV # PCI SR-IOV support -# Floppy drives -device fdc - # ATA controllers device ahci # AHCI-compatible SATA controllers device ata # Legacy ATA/SATA controllers @@ -121,13 +113,9 @@ options AHD_REG_PRETTY_PRINT # Print register bitfields in debug # output. Adds ~215k to driver. device esp # AMD Am53C974 (Tekram DC-390(T)) device hptiop # Highpoint RocketRaid 3xxx series -device isp # Qlogic family -#device ispfw # Firmware for QLogic HBAs- normally a module device mpt # LSI-Logic MPT-Fusion device mps # LSI-Logic MPT-Fusion 2 device mpr # LSI-Logic MPT-Fusion 3 -#device ncr # NCR/Symbios Logic -device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') device trm # Tekram DC395U/UW/F DC315U adapters device adv # Advansys SCSI adapters @@ -151,10 +139,6 @@ device amr # AMI MegaRAID device arcmsr # Areca SATA II RAID device ciss # Compaq Smart RAID 5* device dpt # DPT Smartcache III, IV - See NOTES for options -device hptmv # Highpoint RocketRAID 182x -device hptnr # Highpoint DC7280, R750 -device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx -device hpt27xx # Highpoint RocketRAID 27xx device iir # Intel Integrated RAID device ips # IBM (Adaptec) ServeRAID device mly # Mylex AcceleRAID/eXtremeRAID @@ -167,10 +151,7 @@ device aacp # SCSI passthrough for aac (requires CAM) device aacraid # Adaptec by PMC RAID device ida # Compaq Smart RAID device mfi # LSI MegaRAID SAS -device mlx # Mylex DAC960 family device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s -#XXX pointer/int warnings -#device pst # Promise Supertrak SX6000 device twe # 3ware ATA RAID # NVM Express (NVMe) support @@ -219,7 +200,6 @@ device ppi # Parallel port interface device device puc # Multi I/O cards and multi-channel UARTs # PCI Ethernet NICs. -device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE device de # DEC/Intel DC21x4x (``Tulip'') device em # Intel PRO/1000 Gigabit Ethernet Family device igb # Intel PRO/1000 PCIE Server Gigabit Family @@ -239,7 +219,6 @@ device ae # Attansic/Atheros L2 FastEthernet device age # Attansic/Atheros L1 Gigabit Ethernet device alc # Atheros AR8131/AR8132 Ethernet device ale # Atheros AR8121/AR8113/AR8114 Ethernet -device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet device bfe # Broadcom BCM440x 10/100 Ethernet device bge # Broadcom BCM570xx Gigabit Ethernet device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn @@ -288,14 +267,7 @@ options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later device ath_rate_sample # SampleRate tx rate control for ath #device bwi # Broadcom BCM430x/BCM431x wireless NICs. #device bwn # Broadcom BCM43xx wireless NICs. -device ipw # Intel 2100 wireless NICs. -device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. -device iwn # Intel 4965/1000/5000/6000 wireless NICs. device malo # Marvell Libertas wireless NICs. -device mwl # Marvell 88W8363 802.11n wireless NICs. -device ral # Ralink Technology RT2500 wireless NICs. -device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. -device wpi # Intel 3945ABG wireless NICs. # Pseudo devices. device loop # Network loopback @@ -327,7 +299,6 @@ device umass # Disks/Mass storage - Requires scbus and da # Sound support device sound # Generic sound driver (required) device snd_cmi # CMedia CMI8338/CMI8738 -device snd_csa # Crystal Semiconductor CS461x/428x device snd_emu10kx # Creative SoundBlaster Live! and Audigy device snd_es137x # Ensoniq AudioPCI ES137x device snd_hda # Intel High Definition Audio @@ -348,8 +319,6 @@ device virtio_scsi # VirtIO SCSI device device virtio_balloon # VirtIO Memory Balloon device # HyperV drivers and enchancement support -# NOTE: HYPERV depends on hyperv. They must be added or removed together. -options HYPERV # Hyper-V kernel infrastructure device hyperv # HyperV drivers # Xen HVM Guest Optimizations @@ -360,6 +329,5 @@ device xenpci # Xen HVM Hypervisor services driver # VMware support device vmx # VMware VMXNET3 Ethernet -# Netmap provides direct access to TX/RX rings on supported NICs -device netmap # netmap(4) support - +# The crypto framework is required by IPSEC +device crypto # Required by IPSEC -- 2.11.0 From c3c2f68305751d33ebdc597ab0c4faf7f50f59b5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 27 Mar 2013 11:28:27 +0100 Subject: [PATCH 003/257] Import ELECTRO_BEER Obtained from: ElectroBSD --- sys/amd64/conf/ELECTRO_BEER | 257 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 sys/amd64/conf/ELECTRO_BEER diff --git a/sys/amd64/conf/ELECTRO_BEER b/sys/amd64/conf/ELECTRO_BEER new file mode 100644 index 000000000000..3fb26ab1675f --- /dev/null +++ b/sys/amd64/conf/ELECTRO_BEER @@ -0,0 +1,257 @@ +include ELECTRO_BLOAT + +ident ELECTRO_BEER + +# Debugging for use in -current +nooptions INVARIANTS # Enable calls of extra sanity checking +nooptions INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS +nooptions WITNESS # Enable checks to detect deadlocks and cycles +nooptions WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +nooptions FLOWTABLE # per-cpu routing cache +nooptions XENHVM # Include Xen support + +nooptions SCTP # Stream Control Transmission Protocol + +nodevice fdc + +nodevice ataraid # ATA RAID drives +nodevice atapifd # ATAPI floppy drives +nodevice atapist # ATAPI tape drives + +# ATA controllers +nodevice mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA +nodevice siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA + +# SCSI Controllers +nodevice ahc # AHA2940 and onboard AIC7xxx devices +nooptions AHC_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~128k to driver. +nodevice amd # AMD 53C974 (Tekram DC-390(T)) +nodevice esp # AMD Am53C974 (Tekram DC-390(T)) +nodevice hptiop # Highpoint RocketRaid 3xxx series +nodevice isp # Qlogic family +nodevice ispfw # Firmware for QLogic HBAs- normally a module +nodevice mpt # LSI-Logic MPT-Fusion +nodevice mps # LSI-Logic MPT-Fusion 2 +nodevice ncr # NCR/Symbios Logic +nodevice sym # NCR/Symbios Logic (newer chipsets + those of `ncr') +nodevice trm # Tekram DC395U/UW/F DC315U adapters + +nodevice adv # Advansys SCSI adapters +nodevice adw # Advansys wide SCSI adapters +nodevice aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60. +nodevice bt # Buslogic/Mylex MultiMaster SCSI adapters + +# RAID controllers interfaced to the SCSI subsystem +nodevice amr # AMI MegaRAID +nodevice arcmsr # Areca SATA II RAID +#XXX it is not 64-bit clean, -scottl +nodevice asr # DPT SmartRAID V, VI and Adaptec SCSI RAID +nodevice ciss # Compaq Smart RAID 5* +nodevice dpt # DPT Smartcache III, IV - See NOTES for options +nodevice hptmv # Highpoint RocketRAID 182x +nodevice hptnr # Highpoint DC7280, R750 +nodevice hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx +nodevice hpt27xx # Highpoint RocketRAID 27xx + +nodevice iir # Intel Integrated RAID +nodevice ips # IBM (Adaptec) ServeRAID +nodevice mly # Mylex AcceleRAID/eXtremeRAID +nodevice twa # 3ware 9000 series PATA/SATA RAID +nodevice tws # LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller + +# RAID controllers +nodevice aac # Adaptec FSA RAID +nodevice aacraid # Adaptec by PMC RAID +nodevice aacp # SCSI passthrough for aac (requires CAM) +nodevice ida # Compaq Smart RAID +nodevice mfi # LSI MegaRAID SAS +nodevice mlx # Mylex DAC960 family +#XXX pointer/int warnings +nodevice pst # Promise Supertrak SX6000 +nodevice twe # 3ware ATA RAID + + +# Parallel port +nodevice ppc +nodevice ppbus # Parallel port bus (required) +nodevice lpt # Printer +nodevice plip # TCP/IP over parallel +nodevice ppi # Parallel port interface device +nodevice vpo # Requires scbus and da + +# If you've got a "dumb" serial or parallel PCI card that is +# supported by the puc(4) glue driver, uncomment the following +# line to enable it (connects to sio, uart and/or ppc drivers): +nodevice puc + +# PCI Ethernet NICs. +nodevice bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE +nodevice de # DEC/Intel DC21x4x (``Tulip'') +nodevice em # Intel PRO/1000 Gigabit Ethernet Family +nodevice igb # Intel PRO/1000 PCIE Server Gigabit Family +nodevice ixgbe # Intel PRO/10GbE PCIE Ethernet Family +nodevice le # AMD Am7900 LANCE and Am79C9xx PCnet +nodevice ti # Alteon Networks Tigon I/II gigabit Ethernet +nodevice txp # 3Com 3cR990 (``Typhoon'') +nodevice vx # 3Com 3c590, 3c595 (``Vortex'') + +# PCI Ethernet NICs that use the common MII bus controller code. +# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs! + +nodevice ae # Attansic/Atheros L2 FastEthernet +nodevice age # Attansic/Atheros L1 Gigabit Ethernet +nodevice alc # Atheros AR8131/AR8132 Ethernet +nodevice ale # Atheros AR8121/AR8113/AR8114 Ethernet +nodevice bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet +nodevice bfe # Broadcom BCM440x 10/100 Ethernet +nodevice cas # Sun Cassini/Cassini+ and NS DP83065 Saturn +nodevice dc # DEC/Intel 21143 and various workalikes +nodevice et # Agere ET1310 10/100/Gigabit Ethernet +nodevice fxp # Intel EtherExpress PRO/100B (82557, 82558) +nodevice gem # Sun GEM/Sun ERI/Apple GMAC +nodevice hme # Sun HME (Happy Meal Ethernet) +nodevice jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet +nodevice lge # Level 1 LXT1001 gigabit Ethernet +nodevice msk # Marvell/SysKonnect Yukon II Gigabit Ethernet +nodevice nfe # nVidia nForce MCP on-board Ethernet +nodevice nge # NatSemi DP83820 gigabit Ethernet +nodevice nve # nVidia nForce MCP on-board Ethernet Networking +nodevice pcn # AMD Am79C97x PCI 10/100 (precedence over 'le') +nodevice re # RealTek 8139C+/8169/8169S/8110S +nodevice rl # RealTek 8129/8139 +nodevice sf # Adaptec AIC-6915 (``Starfire'') +nodevice sge # Silicon Integrated Systems SiS190/191 +nodevice sis # Silicon Integrated Systems SiS 900/SiS 7016 +nodevice sk # SysKonnect SK-984x & SK-982x gigabit Ethernet +nodevice ste # Sundance ST201 (D-Link DFE-550TX) +nodevice stge # Sundance/Tamarack TC9021 gigabit Ethernet +nodevice tl # Texas Instruments ThunderLAN +nodevice tx # SMC EtherPower II (83c170 ``EPIC'') +nodevice vge # VIA VT612x gigabit Ethernet +nodevice vr # VIA Rhine, Rhine II +nodevice wb # Winbond W89C840F +nodevice xl # 3Com 3c90x (``Boomerang'', ``Cyclone'') + +# ISA Ethernet NICs. pccard NICs included. +nodevice cs # Crystal Semiconductor CS89x0 NIC +# 'device ed' requires 'device miibus' +nodevice ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards +nodevice ex # Intel EtherExpress Pro/10 and Pro/10+ +nodevice ep # Etherlink III based cards +nodevice fe # Fujitsu MB8696x based cards +nodevice sn # SMC's 9000 series of Ethernet chips +nodevice xe # Xircom pccard Ethernet + +nodevice zyd # ZyDAS zb1211/zb1211b wireless NICs +nodevice urio # Diamond Rio 500 MP3 player + +# Wireless NIC cards +nodevice an # Aironet 4500/4800 802.11 wireless NICs. +nodevice ath # Atheros NIC's +nodevice ath_pci # Atheros pci/cardbus glue +nodevice ath_hal # pci/cardbus chip support +nodevice ath_rate_sample # SampleRate tx rate control for ath +nodevice bwi # Broadcom BCM430x/BCM431x wireless NICs. +nodevice bwn # Broadcom BCM43xx wireless NICs. +nodevice ipw # Intel 2100 wireless NICs. +nodevice iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. +nodevice iwn # Intel 4965/1000/5000/6000 wireless NICs. +nodevice malo # Marvell Libertas wireless NICs. +nodevice mwl # Marvell 88W8363 802.11n wireless NICs. +nodevice ral # Ralink Technology RT2500 wireless NICs. +nodevice wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. +nodevice wpi # Intel 3945ABG wireless NICs. + +# Pseudo devices. +nodevice padlock_rng # VIA Padlock RNG +nodevice rdrand_rng # Intel Bull Mountain RNG +nodevice gif # IPv6 and IPv4 tunneling +nodevice faith # IPv6-to-IPv4 relaying (translation) + +# USB Serial devices +nodevice uark # Technologies ARK3116 based serial adapters +nodevice ubsa # Belkin F5U103 and compatible serial adapters +nodevice uftdi # For FTDI usb serial adapters +nodevice uipaq # Some WinCE based devices +nodevice uplcom # Prolific PL-2303 serial adapters +nodevice uslcom # SI Labs CP2101/CP2102 serial adapters +nodevice uvisor # Visor and Palm devices +nodevice uvscom # USB serial support for DDI pocket's PHS +# USB Ethernet, requires miibus +nodevice aue # ADMtek USB Ethernet +nodevice axe # ASIX Electronics USB Ethernet +nodevice cdce # Generic USB over Ethernet +nodevice cue # CATC USB Ethernet +nodevice kue # Kawasaki LSI USB Ethernet +nodevice rue # RealTek RTL8150 USB Ethernet +nodevice udav # Davicom DM9601E USB + +# USB support + +nodevice uhci # UHCI PCI->USB interface +nodevice ohci # OHCI PCI->USB interface +nodevice ehci # EHCI PCI->USB interface (USB 2.0) +nodevice xhci # XHCI PCI->USB interface (USB 3.0) +nodevice usb # USB Bus (required) +nodevice ukbd # Keyboard +nodevice umass # Disks/Mass storage - Requires scbus and da + +# FireWire support +nodevice firewire # FireWire bus code +nodevice sbp # SCSI over FireWire (Requires scbus and da) +nodevice fwe # Ethernet over FireWire (non-standard!) +nodevice fwip # IP over FireWire (RFC 2734,3146) +nodevice dcons # Dumb console driver +nodevice dcons_crom # Configuration ROM for dcons + +# Sound support +nodevice snd_es137x # Ensoniq AudioPCI ES137x +nodevice snd_ich # Intel, NVidia and other ICH AC'97 Audio +nodevice snd_uaudio # USB Audio +nodevice snd_via8233 # VIA VT8233x Audio +nodevice snd_cmi # CMedia CMI8338/CMI8738 +nodevice snd_csa # Crystal Semiconductor CS461x/428x +nodevice snd_emu10kx # Creative SoundBlaster Live! and Audigy + +# VirtIO support +nodevice virtio # Generic VirtIO bus (required) +nodevice virtio_pci # VirtIO PCI device +nodevice vtnet # VirtIO Ethernet device +nodevice virtio_blk # VirtIO Block device +nodevice virtio_scsi # VirtIO SCSI device +nodevice virtio_balloon # VirtIO Memory Balloon device + +# HyperV drivers +nodevice hyperv # HyperV drivers + +# Xen support +nodevice xenpci # Generic Xen bus + +# VMware support +nodevice vmx # VMware VMXNET3 Ethernet + +# Same for Intel processors +device coretemp + +# man 4 cpuctl +device cpuctl # CPU control pseudo-device + +# UTF-8 in console (8.x+) +options TEKEN_UTF8 + +nodevice netmap + +# Disabling them prevents ZFS from being loaded. +# Should be bisected. +#nooptions NFS_ROOT +#nooptions NFSLOCKD +#nooptions NFSD +#nooptions NFSCL +nodevice ahd +nodevice mpr +nodevice isci +nodevice ses +nodevice mrsas +nodevice ixl +nodevice ixlv -- 2.11.0 From 8607fd68d6420a85067862212315eab5a77e6252 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 14 Apr 2015 17:43:38 +0200 Subject: [PATCH 004/257] Add ELECTRO_BEER for i386 Obtained from: ElectroBSD --- sys/i386/conf/ELECTRO_BEER | 1 + 1 file changed, 1 insertion(+) create mode 120000 sys/i386/conf/ELECTRO_BEER diff --git a/sys/i386/conf/ELECTRO_BEER b/sys/i386/conf/ELECTRO_BEER new file mode 120000 index 000000000000..4a483bb3a8ea --- /dev/null +++ b/sys/i386/conf/ELECTRO_BEER @@ -0,0 +1 @@ +../../amd64/conf/ELECTRO_BEER \ No newline at end of file -- 2.11.0 From dc52b1a919e30b4b745eb420f18c57b9b8b6f6e5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 28 Jun 2015 17:06:35 +0200 Subject: [PATCH 005/257] i386: Copy GENERIC to ELECTRO_BLOAT Obtained from: ElectroBSD --- sys/i386/conf/ELECTRO_BLOAT | 382 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 382 insertions(+) create mode 100644 sys/i386/conf/ELECTRO_BLOAT diff --git a/sys/i386/conf/ELECTRO_BLOAT b/sys/i386/conf/ELECTRO_BLOAT new file mode 100644 index 000000000000..aefc50753f0d --- /dev/null +++ b/sys/i386/conf/ELECTRO_BLOAT @@ -0,0 +1,382 @@ +# +# GENERIC -- Generic kernel configuration file for FreeBSD/i386 +# +# For more information on this file, please read the config(5) manual page, +# and/or the handbook section on Kernel Configuration Files: +# +# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html +# +# The handbook is also available locally in /usr/share/doc/handbook +# if you've installed the doc distribution, otherwise always see the +# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the +# latest information. +# +# An exhaustive list of options and more detailed explanations of the +# device lines is also present in the ../../conf/NOTES and NOTES files. +# If you are in doubt as to the purpose or necessity of a line, check first +# in NOTES. +# +# $FreeBSD$ + +cpu I486_CPU +cpu I586_CPU +cpu I686_CPU +ident GENERIC + +makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols +makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support + +options SCHED_ULE # ULE scheduler +options PREEMPTION # Enable kernel thread preemption +options INET # InterNETworking +options INET6 # IPv6 communications protocols +options IPSEC # IP (v4/v6) security +options TCP_OFFLOAD # TCP offload +options SCTP # Stream Control Transmission Protocol +options FFS # Berkeley Fast Filesystem +options SOFTUPDATES # Enable FFS soft updates support +options UFS_ACL # Support for access control lists +options UFS_DIRHASH # Improve performance on big directories +options UFS_GJOURNAL # Enable gjournal-based UFS journaling +options QUOTA # Enable disk quotas for UFS +options MD_ROOT # MD is a potential root device +options NFSCL # Network Filesystem Client +options NFSD # Network Filesystem Server +options NFSLOCKD # Network Lock Manager +options NFS_ROOT # NFS usable as /, requires NFSCL +options MSDOSFS # MSDOS Filesystem +options CD9660 # ISO 9660 Filesystem +options PROCFS # Process filesystem (requires PSEUDOFS) +options PSEUDOFS # Pseudo-filesystem framework +options GEOM_PART_GPT # GUID Partition Tables. +options GEOM_RAID # Soft RAID functionality. +options GEOM_LABEL # Provides labelization +options COMPAT_FREEBSD4 # Compatible with FreeBSD4 +options COMPAT_FREEBSD5 # Compatible with FreeBSD5 +options COMPAT_FREEBSD6 # Compatible with FreeBSD6 +options COMPAT_FREEBSD7 # Compatible with FreeBSD7 +options COMPAT_FREEBSD9 # Compatible with FreeBSD9 +options COMPAT_FREEBSD10 # Compatible with FreeBSD10 +options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI +options KTRACE # ktrace(1) support +options STACK # stack(9) support +options SYSVSHM # SYSV-style shared memory +options SYSVMSG # SYSV-style message queues +options SYSVSEM # SYSV-style semaphores +options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions +options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed. +options KBD_INSTALL_CDEV # install a CDEV entry in /dev +options HWPMC_HOOKS # Necessary kernel hooks for hwpmc(4) +options AUDIT # Security event auditing +options CAPABILITY_MODE # Capsicum capability mode +options CAPABILITIES # Capsicum capabilities +options MAC # TrustedBSD MAC Framework +options KDTRACE_HOOKS # Kernel DTrace hooks +options DDB_CTF # Kernel ELF linker loads CTF data +options INCLUDE_CONFIG_FILE # Include this file in kernel +options RACCT # Resource accounting framework +options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default +options RCTL # Resource limits + +# Debugging support. Always need this: +options KDB # Enable kernel debugger support. +options KDB_TRACE # Print a stack trace for a panic. +# For full debugger support use (turn off in stable branch): +options DDB # Support DDB. +options GDB # Support remote GDB. +options DEADLKRES # Enable the deadlock resolver +options INVARIANTS # Enable calls of extra sanity checking +options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS +options WITNESS # Enable checks to detect deadlocks and cycles +options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones + +# To make an SMP kernel, the next two lines are needed +options SMP # Symmetric MultiProcessor Kernel +device apic # I/O APIC + +# CPU frequency control +device cpufreq + +# Bus support. +device acpi +device pci +options PCI_HP # PCI-Express native HotPlug +options PCI_IOV # PCI SR-IOV support + +# Floppy drives +device fdc + +# ATA controllers +device ahci # AHCI-compatible SATA controllers +device ata # Legacy ATA/SATA controllers +device mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA +device siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA + +# SCSI Controllers +device ahc # AHA2940 and onboard AIC7xxx devices +options AHC_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~128k to driver. +device ahd # AHA39320/29320 and onboard AIC79xx devices +options AHD_REG_PRETTY_PRINT # Print register bitfields in debug + # output. Adds ~215k to driver. +device esp # AMD Am53C974 (Tekram DC-390(T)) +device hptiop # Highpoint RocketRaid 3xxx series +device isp # Qlogic family +#device ispfw # Firmware for QLogic HBAs- normally a module +device mpt # LSI-Logic MPT-Fusion +device mps # LSI-Logic MPT-Fusion 2 +device mpr # LSI-Logic MPT-Fusion 3 +#device ncr # NCR/Symbios Logic +device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') +device trm # Tekram DC395U/UW/F DC315U adapters + +device adv # Advansys SCSI adapters +device adw # Advansys wide SCSI adapters +device aha # Adaptec 154x SCSI adapters +device aic # Adaptec 15[012]x SCSI adapters, AIC-6[23]60. +device bt # Buslogic/Mylex MultiMaster SCSI adapters + +device ncv # NCR 53C500 +device nsp # Workbit Ninja SCSI-3 +device stg # TMC 18C30/18C50 +device isci # Intel C600 SAS controller + +# ATA/SCSI peripherals +device scbus # SCSI bus (required for ATA/SCSI) +device ch # SCSI media changers +device da # Direct Access (disks) +device sa # Sequential Access (tape etc) +device cd # CD +device pass # Passthrough device (direct ATA/SCSI access) +device ses # Enclosure Services (SES and SAF-TE) +#device ctl # CAM Target Layer + +# RAID controllers interfaced to the SCSI subsystem +device amr # AMI MegaRAID +device arcmsr # Areca SATA II RAID +device ciss # Compaq Smart RAID 5* +device dpt # DPT Smartcache III, IV - See NOTES for options +device hptmv # Highpoint RocketRAID 182x +device hptnr # Highpoint DC7280, R750 +device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx +device hpt27xx # Highpoint RocketRAID 27xx +device iir # Intel Integrated RAID +device ips # IBM (Adaptec) ServeRAID +device mly # Mylex AcceleRAID/eXtremeRAID +device twa # 3ware 9000 series PATA/SATA RAID +device tws # LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller + +# RAID controllers +device aac # Adaptec FSA RAID +device aacp # SCSI passthrough for aac (requires CAM) +device aacraid # Adaptec by PMC RAID +device ida # Compaq Smart RAID +device mfi # LSI MegaRAID SAS +device mlx # Mylex DAC960 family +device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s +device pmspcv # PMC-Sierra SAS/SATA Controller driver +device pst # Promise Supertrak SX6000 +device twe # 3ware ATA RAID + +# NVM Express (NVMe) support +device nvme # base NVMe driver +device nvd # expose NVMe namespace as disks, depends on nvme + +# atkbdc0 controls both the keyboard and the PS/2 mouse +device atkbdc # AT keyboard controller +device atkbd # AT keyboard +device psm # PS/2 mouse + +device kbdmux # keyboard multiplexer + +device vga # VGA video card driver +options VESA # Add support for VESA BIOS Extensions (VBE) + +device splash # Splash screen and screen saver support + +# syscons is the default console driver, resembling an SCO console +device sc +options SC_PIXEL_MODE # add support for the raster text mode + +# vt is the new video console driver +device vt +device vt_vga + +device agp # support several AGP chipsets + +# Power management support (see NOTES for more options) +#device apm +# Add suspend/resume support for the i8254. +device pmtimer + +# PCCARD (PCMCIA) support +# PCMCIA and cardbus bridge support +device cbb # cardbus (yenta) bridge +device pccard # PC Card (16-bit) bus +device cardbus # CardBus (32-bit) bus + +# Serial (COM) ports +device uart # Generic UART driver + +# Parallel port +device ppc +device ppbus # Parallel port bus (required) +device lpt # Printer +device ppi # Parallel port interface device +#device vpo # Requires scbus and da + +device puc # Multi I/O cards and multi-channel UARTs + +# PCI Ethernet NICs. +device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE +device de # DEC/Intel DC21x4x (``Tulip'') +device em # Intel PRO/1000 Gigabit Ethernet Family +device igb # Intel PRO/1000 PCIE Server Gigabit Family +device ixgb # Intel PRO/10GbE Ethernet Card +device le # AMD Am7900 LANCE and Am79C9xx PCnet +device ti # Alteon Networks Tigon I/II gigabit Ethernet +device txp # 3Com 3cR990 (``Typhoon'') +device vx # 3Com 3c590, 3c595 (``Vortex'') + +# PCI Ethernet NICs that use the common MII bus controller code. +# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs! +device miibus # MII bus support +device ae # Attansic/Atheros L2 FastEthernet +device age # Attansic/Atheros L1 Gigabit Ethernet +device alc # Atheros AR8131/AR8132 Ethernet +device ale # Atheros AR8121/AR8113/AR8114 Ethernet +device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet +device bfe # Broadcom BCM440x 10/100 Ethernet +device bge # Broadcom BCM570xx Gigabit Ethernet +device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn +device dc # DEC/Intel 21143 and various workalikes +device et # Agere ET1310 10/100/Gigabit Ethernet +device fxp # Intel EtherExpress PRO/100B (82557, 82558) +device gem # Sun GEM/Sun ERI/Apple GMAC +device hme # Sun HME (Happy Meal Ethernet) +device jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet +device lge # Level 1 LXT1001 gigabit Ethernet +device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet +device nfe # nVidia nForce MCP on-board Ethernet +device nge # NatSemi DP83820 gigabit Ethernet +device pcn # AMD Am79C97x PCI 10/100 (precedence over 'le') +device re # RealTek 8139C+/8169/8169S/8110S +device rl # RealTek 8129/8139 +device sf # Adaptec AIC-6915 (``Starfire'') +device sge # Silicon Integrated Systems SiS190/191 +device sis # Silicon Integrated Systems SiS 900/SiS 7016 +device sk # SysKonnect SK-984x & SK-982x gigabit Ethernet +device ste # Sundance ST201 (D-Link DFE-550TX) +device stge # Sundance/Tamarack TC9021 gigabit Ethernet +device tl # Texas Instruments ThunderLAN +device tx # SMC EtherPower II (83c170 ``EPIC'') +device vge # VIA VT612x gigabit Ethernet +device vr # VIA Rhine, Rhine II +device vte # DM&P Vortex86 RDC R6040 Fast Ethernet +device wb # Winbond W89C840F +device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'') + +# ISA Ethernet NICs. pccard NICs included. +device cs # Crystal Semiconductor CS89x0 NIC +# 'device ed' requires 'device miibus' +device ed # NE[12]000, SMC Ultra, 3c503, DS8390 cards +device ex # Intel EtherExpress Pro/10 and Pro/10+ +device ep # Etherlink III based cards +device fe # Fujitsu MB8696x based cards +device ie # EtherExpress 8/16, 3C507, StarLAN 10 etc. +device sn # SMC's 9000 series of Ethernet chips +device xe # Xircom pccard Ethernet + +# Wireless NIC cards +device wlan # 802.11 support +options IEEE80211_DEBUG # enable debug msgs +options IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's +options IEEE80211_SUPPORT_MESH # enable 802.11s draft support +device wlan_wep # 802.11 WEP support +device wlan_ccmp # 802.11 CCMP support +device wlan_tkip # 802.11 TKIP support +device wlan_amrr # AMRR transmit rate control algorithm +device an # Aironet 4500/4800 802.11 wireless NICs. +device ath # Atheros NICs +device ath_pci # Atheros pci/cardbus glue +device ath_hal # pci/cardbus chip support +options AH_SUPPORT_AR5416 # enable AR5416 tx/rx descriptors +options AH_AR5416_INTERRUPT_MITIGATION # AR5416 interrupt mitigation +options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later +device ath_rate_sample # SampleRate tx rate control for ath +#device bwi # Broadcom BCM430x/BCM431x wireless NICs. +#device bwn # Broadcom BCM43xx wireless NICs. +device ipw # Intel 2100 wireless NICs. +device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. +device iwn # Intel 4965/1000/5000/6000 wireless NICs. +device malo # Marvell Libertas wireless NICs. +device mwl # Marvell 88W8363 802.11n wireless NICs. +device ral # Ralink Technology RT2500 wireless NICs. +device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. +#device wl # Older non 802.11 Wavelan wireless NIC. +device wpi # Intel 3945ABG wireless NICs. + +# Pseudo devices. +device loop # Network loopback +device random # Entropy device +device padlock_rng # VIA Padlock RNG +device rdrand_rng # Intel Bull Mountain RNG +device ether # Ethernet support +device vlan # 802.1Q VLAN support +device tun # Packet tunnel. +device md # Memory "disks" +device gif # IPv6 and IPv4 tunneling +device firmware # firmware assist module + +# The `bpf' device enables the Berkeley Packet Filter. +# Be aware of the administrative consequences of enabling this! +# Note that 'bpf' is required for DHCP. +device bpf # Berkeley packet filter + +# USB support +options USB_DEBUG # enable debug msgs +device uhci # UHCI PCI->USB interface +device ohci # OHCI PCI->USB interface +device ehci # EHCI PCI->USB interface (USB 2.0) +device xhci # XHCI PCI->USB interface (USB 3.0) +device usb # USB Bus (required) +device ukbd # Keyboard +device umass # Disks/Mass storage - Requires scbus and da + +# Sound support +device sound # Generic sound driver (required) +device snd_cmi # CMedia CMI8338/CMI8738 +device snd_csa # Crystal Semiconductor CS461x/428x +device snd_emu10kx # Creative SoundBlaster Live! and Audigy +device snd_es137x # Ensoniq AudioPCI ES137x +device snd_hda # Intel High Definition Audio +device snd_ich # Intel, NVidia and other ICH AC'97 Audio +device snd_via8233 # VIA VT8233x Audio + +# MMC/SD +device mmc # MMC/SD bus +device mmcsd # MMC/SD memory card +device sdhci # Generic PCI SD Host Controller + +# VirtIO support +device virtio # Generic VirtIO bus (required) +device virtio_pci # VirtIO PCI device +device vtnet # VirtIO Ethernet device +device virtio_blk # VirtIO Block device +device virtio_scsi # VirtIO SCSI device +device virtio_balloon # VirtIO Memory Balloon device + +# HyperV drivers and enchancement support +device hyperv # HyperV drivers + +# Xen HVM Guest Optimizations +# NOTE: XENHVM depends on xenpci. They must be added or removed together. +options XENHVM # Xen HVM kernel infrastructure +device xenpci # Xen HVM Hypervisor services driver + +# VMware support +device vmx # VMware VMXNET3 Ethernet + +# The crypto framework is required by IPSEC +device crypto # Required by IPSEC -- 2.11.0 From 98e297c54599fbdf02781ad6207b6d9c74b77f85 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 28 Jun 2015 17:55:42 +0200 Subject: [PATCH 006/257] Adjust shiny new ELECTRO_BLOAT i386 for ElectroBSD Obtained from: ElectroBSD --- sys/i386/conf/ELECTRO_BLOAT | 81 ++++++++++----------------------------------- 1 file changed, 17 insertions(+), 64 deletions(-) diff --git a/sys/i386/conf/ELECTRO_BLOAT b/sys/i386/conf/ELECTRO_BLOAT index aefc50753f0d..fcfb7c314411 100644 --- a/sys/i386/conf/ELECTRO_BLOAT +++ b/sys/i386/conf/ELECTRO_BLOAT @@ -1,27 +1,11 @@ -# -# GENERIC -- Generic kernel configuration file for FreeBSD/i386 -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# -# $FreeBSD$ - -cpu I486_CPU -cpu I586_CPU +# ELECTRO_BLOAT -- Modified copy of the GENERIC kernel configuration file +# Used for the release media. + cpu I686_CPU -ident GENERIC +ident ELECTRO_BLOAT + +# Hacker-movie-compatible colors +options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK) makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support @@ -30,9 +14,7 @@ options SCHED_ULE # ULE scheduler options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols -options IPSEC # IP (v4/v6) security options TCP_OFFLOAD # TCP offload -options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support options UFS_ACL # Support for access control lists @@ -49,12 +31,7 @@ options CD9660 # ISO 9660 Filesystem options PROCFS # Process filesystem (requires PSEUDOFS) options PSEUDOFS # Pseudo-filesystem framework options GEOM_PART_GPT # GUID Partition Tables. -options GEOM_RAID # Soft RAID functionality. options GEOM_LABEL # Provides labelization -options COMPAT_FREEBSD4 # Compatible with FreeBSD4 -options COMPAT_FREEBSD5 # Compatible with FreeBSD5 -options COMPAT_FREEBSD6 # Compatible with FreeBSD6 -options COMPAT_FREEBSD7 # Compatible with FreeBSD7 options COMPAT_FREEBSD9 # Compatible with FreeBSD9 options COMPAT_FREEBSD10 # Compatible with FreeBSD10 options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI @@ -85,10 +62,6 @@ options KDB_TRACE # Print a stack trace for a panic. options DDB # Support DDB. options GDB # Support remote GDB. options DEADLKRES # Enable the deadlock resolver -options INVARIANTS # Enable calls of extra sanity checking -options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS -options WITNESS # Enable checks to detect deadlocks and cycles -options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # To make an SMP kernel, the next two lines are needed @@ -101,15 +74,12 @@ device cpufreq # Bus support. device acpi device pci -options PCI_HP # PCI-Express native HotPlug options PCI_IOV # PCI SR-IOV support -# Floppy drives -device fdc - # ATA controllers device ahci # AHCI-compatible SATA controllers device ata # Legacy ATA/SATA controllers +options ATA_STATIC_ID # Static device numbering device mvs # Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA device siis # SiliconImage SiI3124/SiI3132/SiI3531 SATA @@ -123,12 +93,9 @@ options AHD_REG_PRETTY_PRINT # Print register bitfields in debug device esp # AMD Am53C974 (Tekram DC-390(T)) device hptiop # Highpoint RocketRaid 3xxx series device isp # Qlogic family -#device ispfw # Firmware for QLogic HBAs- normally a module device mpt # LSI-Logic MPT-Fusion device mps # LSI-Logic MPT-Fusion 2 device mpr # LSI-Logic MPT-Fusion 3 -#device ncr # NCR/Symbios Logic -device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') device trm # Tekram DC395U/UW/F DC315U adapters device adv # Advansys SCSI adapters @@ -157,10 +124,6 @@ device amr # AMI MegaRAID device arcmsr # Areca SATA II RAID device ciss # Compaq Smart RAID 5* device dpt # DPT Smartcache III, IV - See NOTES for options -device hptmv # Highpoint RocketRAID 182x -device hptnr # Highpoint DC7280, R750 -device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx -device hpt27xx # Highpoint RocketRAID 27xx device iir # Intel Integrated RAID device ips # IBM (Adaptec) ServeRAID device mly # Mylex AcceleRAID/eXtremeRAID @@ -173,16 +136,10 @@ device aacp # SCSI passthrough for aac (requires CAM) device aacraid # Adaptec by PMC RAID device ida # Compaq Smart RAID device mfi # LSI MegaRAID SAS -device mlx # Mylex DAC960 family device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s -device pmspcv # PMC-Sierra SAS/SATA Controller driver device pst # Promise Supertrak SX6000 device twe # 3ware ATA RAID -# NVM Express (NVMe) support -device nvme # base NVMe driver -device nvd # expose NVMe namespace as disks, depends on nvme - # atkbdc0 controls both the keyboard and the PS/2 mouse device atkbdc # AT keyboard controller device atkbd # AT keyboard @@ -205,8 +162,6 @@ device vt_vga device agp # support several AGP chipsets -# Power management support (see NOTES for more options) -#device apm # Add suspend/resume support for the i8254. device pmtimer @@ -229,7 +184,6 @@ device ppi # Parallel port interface device device puc # Multi I/O cards and multi-channel UARTs # PCI Ethernet NICs. -device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE device de # DEC/Intel DC21x4x (``Tulip'') device em # Intel PRO/1000 Gigabit Ethernet Family device igb # Intel PRO/1000 PCIE Server Gigabit Family @@ -246,7 +200,6 @@ device ae # Attansic/Atheros L2 FastEthernet device age # Attansic/Atheros L1 Gigabit Ethernet device alc # Atheros AR8131/AR8132 Ethernet device ale # Atheros AR8121/AR8113/AR8114 Ethernet -device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet device bfe # Broadcom BCM440x 10/100 Ethernet device bge # Broadcom BCM570xx Gigabit Ethernet device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn @@ -307,15 +260,7 @@ options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later device ath_rate_sample # SampleRate tx rate control for ath #device bwi # Broadcom BCM430x/BCM431x wireless NICs. #device bwn # Broadcom BCM43xx wireless NICs. -device ipw # Intel 2100 wireless NICs. -device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. -device iwn # Intel 4965/1000/5000/6000 wireless NICs. device malo # Marvell Libertas wireless NICs. -device mwl # Marvell 88W8363 802.11n wireless NICs. -device ral # Ralink Technology RT2500 wireless NICs. -device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. -#device wl # Older non 802.11 Wavelan wireless NIC. -device wpi # Intel 3945ABG wireless NICs. # Pseudo devices. device loop # Network loopback @@ -347,7 +292,6 @@ device umass # Disks/Mass storage - Requires scbus and da # Sound support device sound # Generic sound driver (required) device snd_cmi # CMedia CMI8338/CMI8738 -device snd_csa # Crystal Semiconductor CS461x/428x device snd_emu10kx # Creative SoundBlaster Live! and Audigy device snd_es137x # Ensoniq AudioPCI ES137x device snd_hda # Intel High Definition Audio @@ -380,3 +324,12 @@ device vmx # VMware VMXNET3 Ethernet # The crypto framework is required by IPSEC device crypto # Required by IPSEC + +# This used to be required for ZFS when compiled with clang. +# For details see UPDATING entry 20121223. After r286288 it's +# probably no longer necessary, but for now we keep it anyway. +options KSTACK_PAGES=4 + +# Increase the size of the kernel virtual address space +# so ZFS can cache more stuff. +options KVA_PAGES=512 -- 2.11.0 From de4ae8f4807717f4d748685911065bc50d5849fa Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 31 May 2015 17:38:09 +0200 Subject: [PATCH 007/257] Change amd64 default KERNCONF to ELECTRO_BLOAT Obtained from: ElectroBSD --- Makefile.inc1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.inc1 b/Makefile.inc1 index f6813bfab744..7cf5d56df54a 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1133,6 +1133,8 @@ KERNCONF=${KERNFAST} .endif .if ${TARGET_ARCH} == "powerpc64" KERNCONF?= GENERIC64 +.elif ${TARGET_ARCH} == "amd64" +KERNCONF?= ELECTRO_BLOAT .else KERNCONF?= GENERIC .endif -- 2.11.0 From a3e20f577887b2e55127e4858d440df0f51e34c8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 22 May 2016 12:10:03 +0200 Subject: [PATCH 008/257] Use ELECTRO_BLOAT as default on i386 Obtained from: ElectroBSD --- Makefile.inc1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index 7cf5d56df54a..967325f8161c 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1133,7 +1133,7 @@ KERNCONF=${KERNFAST} .endif .if ${TARGET_ARCH} == "powerpc64" KERNCONF?= GENERIC64 -.elif ${TARGET_ARCH} == "amd64" +.elif ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" KERNCONF?= ELECTRO_BLOAT .else KERNCONF?= GENERIC -- 2.11.0 From 6e9e8b55086885bd86ce2ee1eef8eb2ffdaba6f5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 4 Aug 2011 19:19:51 +0200 Subject: [PATCH 009/257] Register the product id for Feiya Memory Bar Obtained from: ElectroBSD --- sys/dev/usb/usbdevs | 1 + 1 file changed, 1 insertion(+) diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs index fc898ffca6e9..1c6cf7620d27 100644 --- a/sys/dev/usb/usbdevs +++ b/sys/dev/usb/usbdevs @@ -1867,6 +1867,7 @@ product FALCOM SAMBA 0x0005 FTDI compatible adapter /* FEIYA products */ product FEIYA DUMMY 0x0000 Dummy product +product FEIYA MEMORY_BAR 0x1000 Memory Bar product FEIYA 5IN1 0x1132 5-in-1 Card Reader product FEIYA ELANGO 0x6200 MicroSDHC Card Reader product FEIYA AC110 0x6300 AC-110 Card Reader -- 2.11.0 From 08195fd782f27342ce125b485a3486e02def4c01 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 4 Aug 2011 19:22:46 +0200 Subject: [PATCH 010/257] Add the full name of 'Feya Technology Corp.' Obtained from: ElectroBSD --- sys/dev/usb/usbdevs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs index 1c6cf7620d27..93d7a7a88cd0 100644 --- a/sys/dev/usb/usbdevs +++ b/sys/dev/usb/usbdevs @@ -446,7 +446,7 @@ vendor DIGIANSWER 0x08fd Digianswer vendor AUTHENTEC 0x08ff AuthenTec vendor AUDIOTECHNICA 0x0909 Audio-Technica vendor TRUMPION 0x090a Trumpion Microelectronics -vendor FEIYA 0x090c Feiya +vendor FEIYA 0x090c Feiya Technology Corp. vendor ALATION 0x0910 Alation Systems vendor GLOBESPAN 0x0915 Globespan vendor CONCORDCAMERA 0x0919 Concord Camera -- 2.11.0 From f404f133c2e4c0132bd8d4a54ca27f4a7471687a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 19 Feb 2013 14:42:00 +0100 Subject: [PATCH 011/257] Let g_eli_*read_done() deal with lost devices without causing panics Seems to fix kern/162036 for me. Obtained from: ElectroBSD --- sys/geom/eli/g_eli.c | 3 ++- sys/geom/eli/g_eli_privacy.c | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index 6d734aece183..bd198c043c2c 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -214,7 +214,8 @@ g_eli_read_done(struct bio *bp) pbp->bio_driver2 = NULL; } g_io_deliver(pbp, pbp->bio_error); - atomic_subtract_int(&sc->sc_inflight, 1); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); return; } mtx_lock(&sc->sc_queue_mtx); diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c index d636e1fb0ba0..6ed584684092 100644 --- a/sys/geom/eli/g_eli_privacy.c +++ b/sys/geom/eli/g_eli_privacy.c @@ -87,7 +87,8 @@ g_eli_crypto_read_done(struct cryptop *crp) bp->bio_error = crp->crp_etype; } sc = bp->bio_to->geom->softc; - g_eli_key_drop(sc, crp->crp_desc->crd_key); + if (sc != NULL) + g_eli_key_drop(sc, crp->crp_desc->crd_key); /* * Do we have all sectors already? */ @@ -104,7 +105,8 @@ g_eli_crypto_read_done(struct cryptop *crp) * Read is finished, send it up. */ g_io_deliver(bp, bp->bio_error); - atomic_subtract_int(&sc->sc_inflight, 1); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); return (0); } -- 2.11.0 From 1f8da7687b8a3fd6cad378dd58b75f0ded57a43c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 22 Feb 2013 11:09:20 +0100 Subject: [PATCH 012/257] Let g_eli_write_done() verify that sc isn't NULL as well It looks similar enough to g_eli_read_done() to be affected by kern/162036 as well. Obtained from: ElectroBSD --- sys/geom/eli/g_eli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index bd198c043c2c..920ed0cc118e 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -260,7 +260,8 @@ g_eli_write_done(struct bio *bp) */ sc = pbp->bio_to->geom->softc; g_io_deliver(pbp, pbp->bio_error); - atomic_subtract_int(&sc->sc_inflight, 1); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); } /* -- 2.11.0 From 3fcd5e8525fb0fa0c1e1d6d7266a8c416804b5d9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 20 Dec 2013 18:45:00 +0100 Subject: [PATCH 013/257] Let g_eli_keyfiles_load() log the size of the keyfile loaded Obtained from: ElectroBSD --- sys/geom/eli/g_eli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index 920ed0cc118e..9add13044a6d 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -959,8 +959,8 @@ g_eli_keyfiles_load(struct hmac_ctx *ctx, const char *provider) name); return (0); } - G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s).", file, - provider, name); + G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s) (size: %d).", file, + provider, name, (unsigned)size); g_eli_crypto_hmac_update(ctx, data, size); } } -- 2.11.0 From 83ee7c119f12a71ac055cc40fc1877127406b9b4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 13:27:28 +0100 Subject: [PATCH 014/257] ggated: Ignore SIGPIPE to prevent DoS ... by a single prematurely closed client connection. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index e234cb57b290..36a8d5203b21 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1025,6 +1025,7 @@ main(int argc, char *argv[]) pidfile_write(pfh); signal(SIGCHLD, SIG_IGN); + signal(SIGPIPE, SIG_IGN); sfd = socket(AF_INET, SOCK_STREAM, 0); if (sfd == -1) -- 2.11.0 From b77bbffe7b4efcb1eeb6f2da9f16e1ba8ecb3053 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 13:47:31 +0100 Subject: [PATCH 015/257] ggated: Remove connection if the initial packet couldn't be sent Should help to mitigate DoS after flooding ggated with incomplete requests: error: accept(): Software caused connection abort. error: Exiting. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 36a8d5203b21..7574e20d3480 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -928,6 +928,7 @@ handshake(struct sockaddr *from, int sfd) if (data == -1) { sendfail(sfd, errno, "Error while sending initial packet: %s.", strerror(errno)); + connection_remove(conn); return (0); } -- 2.11.0 From 4b486daa0e44107a20b2d63d49e2acd64ba69957 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 14:09:24 +0100 Subject: [PATCH 016/257] ggated: Continue if accept() is interrupted or the remote connection is lost Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 7574e20d3480..fc7c86ae75ac 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1050,9 +1050,11 @@ main(int argc, char *argv[]) for (;;) { fromlen = sizeof(from); tmpsfd = accept(sfd, &from, &fromlen); - if (tmpsfd == -1) + if (tmpsfd == -1) { + if (errno == EINTR || errno == ECONNABORTED) + continue; g_gate_xlog("accept(): %s.", strerror(errno)); - + } if (got_sighup) { got_sighup = 0; exports_get(); -- 2.11.0 From d75ae4968e55c6bc22c527a5174b0e0678c5e20a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 15:18:27 +0100 Subject: [PATCH 017/257] ggated: Initialize conn->c_diskfd in connection_new() Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index fc7c86ae75ac..9d523459623d 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -453,7 +453,7 @@ connection_new(struct g_gate_cinit *cinit, struct sockaddr *s, int sfd) conn->c_token = cinit->gc_token; ip = htonl(((struct sockaddr_in *)(void *)s)->sin_addr.s_addr); conn->c_srcip = ip; - conn->c_sendfd = conn->c_recvfd = -1; + conn->c_sendfd = conn->c_recvfd = conn->c_diskfd = -1; if ((cinit->gc_flags & GGATE_FLAG_SEND) != 0) conn->c_sendfd = sfd; else -- 2.11.0 From 3c4974216fab0d21329716ca001d819b2b237ae5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 14:16:47 +0100 Subject: [PATCH 018/257] ggated: Prevent c_diskfd leaks through connection_remove() Should help against DoS: [...] debug: Connection created [127.0.0.1, /tank/scratch/testfile]. debug: New connection created (token=2197914058). debug: exports[/tank/scratch/testfile2]: Path mismatch. debug: Sending initial packet. error: accept(): Too many open files. error: Exiting. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 9d523459623d..6d5521804e60 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -512,6 +512,8 @@ connection_remove(struct ggd_connection *conn) close(conn->c_sendfd); if (conn->c_recvfd != -1) close(conn->c_recvfd); + if (conn->c_diskfd != -1) + close(conn->c_diskfd); free(conn->c_path); free(conn); } -- 2.11.0 From c1fa50b00f71a55c7008db20923136e6284a9fe7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Dec 2014 15:52:39 +0100 Subject: [PATCH 019/257] ggated: Check for connection_add() failures properly Prevents a socket leak Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 6d5521804e60..6236c50a4359 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -890,7 +890,7 @@ handshake(struct sockaddr *from, int sfd) */ g_gate_log(LOG_DEBUG, "Found existing connection (token=%lu).", (unsigned long)conn->c_token); - if (connection_add(conn, &cinit, from, sfd) == -1) { + if (connection_add(conn, &cinit, from, sfd) == EEXIST) { connection_remove(conn); return (0); } -- 2.11.0 From f4ee506a76f3f71aada7acf714b3d0197dddf7b9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 8 Dec 2014 17:59:38 +0100 Subject: [PATCH 020/257] ggated: Do not leak stack data in sendfail() Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 6236c50a4359..4bb1f7c6004a 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -591,6 +591,7 @@ sendfail(int sfd, int error, const char *fmt, ...) va_list ap; ssize_t data; + bzero(&sinit, sizeof(sinit)); sinit.gs_error = error; g_gate_swap2n_sinit(&sinit); data = g_gate_send(sfd, &sinit, sizeof(sinit), 0); -- 2.11.0 From a05b3aac58b2518f908fcfd254fb377ee9ad3e37 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 2 Apr 2015 15:24:58 +0200 Subject: [PATCH 021/257] ggated recv_thread(): Do not queue incomplete WRITE requests Verifying that g_gate_recv()'s return code isn't -1 is insufficient as it's a thin wrapper arround recv(2) which, quoting its man page, "may still return less data than requested if a signal is caught, an error or disconnect occurs, or the next data to be received is of a different type than that returned". Previously incomplete WRITE requests would be scheduled with partially uninitialized memory, potentially resulting in file system corruption or, worse, bogus data being later on returned as valid. Security impact: A MITM may cause data corruption by disrupting the connection from ggatec's send_thread() to ggated's recv_thread() at the right point in time. This does not require access to the plain text traffic but if encryption is involved the attacker would have to guess that it's ggate traffic and disrupt connections blindly, hoping that some of the disruptions trigger the bug. The issue was discovered after ZFS on the ggatec side reported checksum errors which weren't reproducible on the ggated side where ZFS had received and checksummed bogus data. The ggate traffic was tunneled through SSH and Tor with sshd running as Tor location hidden service. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 4bb1f7c6004a..9883355d81f1 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -669,6 +669,9 @@ recv_thread(void *arg) if (data == -1) { g_gate_xlog("Error while receiving data: %s.", strerror(errno)); + } else if ((uint32_t)data != req->r_length) { + g_gate_xlog("Received %d bytes of data while " + "expecting %u.", data, req->r_length); } } -- 2.11.0 From b40d5dc37c179fadc5ffe3b4cb8f2112491fedef Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 2 Apr 2015 12:09:40 +0200 Subject: [PATCH 022/257] ggated recv_thread(): Do not queue requests with invalid values ... that would cause abort()s when read by the disk_thread() later on. From ggatec's point of view it doesn't make a difference as the connection will get closed either way, but at least the admin on the server side doesn't have to deal with core dumps. Security impact: An authenticated attacker may intentionally cause the ggated process that handles the attacker's connection to core dump and thus use more disk space than intentionally provisioned by the server admin. Without the following patch ggated core dumps may require more than 100 GB of disk space. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 9883355d81f1..6b53decdeb65 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -654,6 +654,23 @@ recv_thread(void *arg) (intmax_t)req->r_offset, (unsigned)req->r_length); /* + * Reject requests that violate assertions in disk_thread(). + */ + if (req->r_cmd != GGATE_CMD_READ && + req->r_cmd != GGATE_CMD_WRITE) { + g_gate_xlog("Request contains invalid command."); + } + if (req->r_offset + req->r_length > + (uintmax_t)conn->c_mediasize) { + g_gate_xlog("Request out of bounds."); + } + if (req->r_offset % conn->c_sectorsize != 0 || + req->r_length % conn->c_sectorsize != 0) { + g_gate_xlog("Request length or offset does " + "not fit sector size."); + } + + /* * Allocate memory for data. */ req->r_data = malloc_waitok(req->r_length); -- 2.11.0 From f677cfb0ca9596d3f2173bb8e47b601bf3dcd57f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 2 Apr 2015 19:52:54 +0200 Subject: [PATCH 023/257] ggated recv_thread(): Reject request with more than MAXPHYS bytes of data .. to limit the amount of memory we (try to) allocate on behalf of the client without knowing whether or not the client actually intents to use it. MAXPHYS is the hardcoded limit in ggatec so anything above it is suspicious and could be a DoS attempt. This commit forces users who like to tune MAXPHYS to make sure the value used by ggated is not below the one used by ggatec. While not ideal, this seems preferable to the DoS risk. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 6b53decdeb65..22810b0a00fd 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -671,6 +671,16 @@ recv_thread(void *arg) } /* + * Limit the amount of memory we allocate on behalf of + * the client. MAXPHYS is the hard limit in ggatec, + * values above it are thus pretty suspicious. + */ + if (req->r_length > MAXPHYS) { + g_gate_xlog("Request length above MAXPHYS: %u > %u", + (unsigned)req->r_length, MAXPHYS); + } + + /* * Allocate memory for data. */ req->r_data = malloc_waitok(req->r_length); -- 2.11.0 From 10175befe111e8ccd79c9aa99c70e79bc8900073 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 24 Apr 2015 14:04:31 +0200 Subject: [PATCH 024/257] ggatec: Add support for SOCKS5 with domain names Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 104 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 5 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ea4f701abafa..80a653dff6ba 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -64,6 +64,8 @@ static unsigned flags = 0; static int force = 0; static unsigned queue_size = G_GATE_QUEUE_SIZE; static unsigned port = G_GATE_PORT; +static char *socks_dest = NULL; +static unsigned dest_port = 3080; static off_t mediasize; static unsigned sectorsize = 0; static unsigned timeout = G_GATE_TIMEOUT; @@ -78,9 +80,11 @@ usage(void) fprintf(stderr, "usage: %s create [-nv] [-o ] [-p port] " "[-q queue_size] [-R rcvbuf] [-S sndbuf] [-s sectorsize] " - "[-t timeout] [-u unit] \n", getprogname()); + "[-t timeout] [-T :] [-u unit] \n", + getprogname()); fprintf(stderr, " %s rescue [-nv] [-o ] [-p port] " - "[-R rcvbuf] [-S sndbuf] <-u unit> \n", getprogname()); + "[-R rcvbuf] [-S sndbuf] [-T :] <-u unit> " + " \n", getprogname()); fprintf(stderr, " %s destroy [-f] <-u unit>\n", getprogname()); fprintf(stderr, " %s list [-v] [-u unit]\n", getprogname()); exit(EXIT_FAILURE); @@ -238,6 +242,69 @@ recv_thread(void *arg __unused) pthread_exit(NULL); } +static void +negotiate_socks_connection(int sfd) +{ + struct negotiation_request { + char version; + char nmethods; + char method; + } neg_request; + struct socks_request { + char version; + char cmd; + char reserved; + char address_type; + char host_length; + char dest[255 + 2]; + } socks_request; + char response[10]; + size_t request_length; + size_t host_length; + + host_length = strlen(socks_dest); + + neg_request.version = '\x05'; + neg_request.nmethods = '\x01'; /* We support one method: */ + neg_request.method = '\x00'; /* no authentication */ + + g_gate_log(LOG_DEBUG, "Starting SOCKS negotiation."); + if (g_gate_send(sfd, &neg_request, sizeof(neg_request), MSG_NOSIGNAL) == -1) + g_gate_xlog("Failed to send SOCKS negotiation request."); + + if (g_gate_recv(sfd, &response, sizeof(response), MSG_WAITALL) != 2) + g_gate_xlog("Failed to read SOCKS negotiation response."); + + if (response[0] != '\x05' || response[1] != '\x00') + g_gate_xlog("SOCKS negotiation failed."); + + g_gate_log(LOG_DEBUG, "Negotiated SOCKS5. " + "Requesting connection to %s:%d.", socks_dest, dest_port); + + socks_request.version = '\x05'; + socks_request.cmd = '\x01'; /* Connect */ + socks_request.reserved = '\x00'; + socks_request.address_type = '\x03'; /* Address is domain name */; + socks_request.host_length = (char)host_length; + strncpy(socks_request.dest, socks_dest, host_length); + socks_request.dest[host_length] = (char)((dest_port >> 8) & 0xff); + socks_request.dest[host_length + 1] = (char)(dest_port & 0xff); + request_length = sizeof(socks_request) - sizeof(socks_request.dest) + + host_length + 2; + + if (g_gate_send(sfd, &socks_request, request_length, MSG_NOSIGNAL) == -1) + g_gate_xlog("Failed to send SOCKS5 request."); + + if (g_gate_recv(sfd, &response, sizeof(response), MSG_WAITALL) != sizeof(response)) + g_gate_xlog("Failed to read SOCKS5 response."); + + if (response[0] != '\x05' || response[1] != '\x00') + g_gate_xlog("Failed to SOCKS5 connect to %s:%d", + socks_dest, dest_port); + + g_gate_log(LOG_INFO, "Connected to: %s:%d.", socks_dest, dest_port); +} + static int handshake(int dir) { @@ -276,6 +343,9 @@ handshake(int dir) g_gate_log(LOG_INFO, "Connected to the server: %s:%d.", host, port); + if (socks_dest != NULL) + negotiate_socks_connection(sfd); + /* * Create and send version packet. */ @@ -455,8 +525,13 @@ g_gatec_create(void) ggioc.gctl_maxcount = queue_size; ggioc.gctl_timeout = timeout; ggioc.gctl_unit = unit; - snprintf(ggioc.gctl_info, sizeof(ggioc.gctl_info), "%s:%u %s", host, - port, path); + if (socks_dest != NULL) + snprintf(ggioc.gctl_info, sizeof(ggioc.gctl_info), + "socks5://%s:%u -> %s:%u %s", host, + port, socks_dest, dest_port, path); + else + snprintf(ggioc.gctl_info, sizeof(ggioc.gctl_info), "%s:%u %s", + host, port, path); g_gate_ioctl(G_GATE_CMD_CREATE, &ggioc); if (unit == -1) { printf("%s%u\n", G_GATE_PROVIDER_NAME, ggioc.gctl_unit); @@ -505,8 +580,9 @@ main(int argc, char *argv[]) argv += 1; for (;;) { int ch; + char *p; - ch = getopt(argc, argv, "fno:p:q:R:S:s:t:u:v"); + ch = getopt(argc, argv, "fno:p:q:R:S:s:t:T:u:v"); if (ch == -1) break; switch (ch) { @@ -574,6 +650,24 @@ main(int argc, char *argv[]) if (sectorsize == 0 && errno != 0) errx(EXIT_FAILURE, "Invalid sectorsize."); break; + case 'T': + if (action != CREATE && action != RESCUE) + usage(); + socks_dest = optarg; + p = strchr(socks_dest, ':'); + if (p != NULL) { + errno = 0; + *p = '\0'; + p++; + dest_port = strtoul(p, NULL, 10); + if (dest_port == 0 && errno != 0) + errx(EXIT_FAILURE, + "Invalid socks5t port: %s.", p); + } + if (strlen(socks_dest) > (size_t)255) + errx(EXIT_FAILURE, + "Socks destination address too long."); + break; case 't': if (action != CREATE) usage(); -- 2.11.0 From 07b82267f9bbfb3edbbcfb3dd0bdeea62896054b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 16:44:32 +0200 Subject: [PATCH 025/257] ggatec: Deduplicate information in debug output Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 80a653dff6ba..ed5b94915ac6 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -176,8 +176,9 @@ send_thread(void *arg __unused) pthread_kill(recvtd, SIGUSR1); break; } - g_gate_log(LOG_DEBUG, "Sent %zd bytes (offset=%llu, " - "size=%u).", data, hdr.gh_offset, hdr.gh_length); + g_gate_log(LOG_DEBUG, "Sent data packet " + "(offset=%llu, size=%u).", + hdr.gh_offset, hdr.gh_length); } } g_gate_log(LOG_DEBUG, "%s: Died.", __func__); @@ -224,16 +225,15 @@ recv_thread(void *arg __unused) ggio.gctl_length, MSG_WAITALL); if (reconnect) break; - g_gate_log(LOG_DEBUG, "Received data packet."); if (data != ggio.gctl_length) { g_gate_log(LOG_ERR, "Lost connection 4."); reconnect = 1; pthread_kill(sendtd, SIGUSR1); break; } - g_gate_log(LOG_DEBUG, "Received %d bytes (offset=%ju, " - "size=%zu).", data, (uintmax_t)hdr.gh_offset, - (size_t)hdr.gh_length); + g_gate_log(LOG_DEBUG, "Received data packet " + "(offset=%ju, size=%zu).", + (uintmax_t)hdr.gh_offset, (size_t)hdr.gh_length); } g_gate_ioctl(G_GATE_CMD_DONE, &ggio); -- 2.11.0 From a2958967254b1c7d2ab0a42e90a18e80cd1db371 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 24 Apr 2015 15:26:42 +0200 Subject: [PATCH 026/257] ggatec.8: Document SOCKS5 support Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.8 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.8 b/sbin/ggate/ggatec/ggatec.8 index 705d42f657c2..c50416f4e418 100644 --- a/sbin/ggate/ggatec/ggatec.8 +++ b/sbin/ggate/ggatec/ggatec.8 @@ -41,6 +41,7 @@ .Op Fl R Ar rcvbuf .Op Fl S Ar sndbuf .Op Fl s Ar sectorsize +.Op Fl T Ar remote_target:port .Op Fl t Ar timeout .Op Fl u Ar unit .Ar host @@ -53,6 +54,7 @@ .Op Fl p Ar port .Op Fl R Ar rcvbuf .Op Fl S Ar sndbuf +.Op Fl T Ar remote_target:port .Fl u Ar unit .Ar host .Ar path @@ -137,6 +139,9 @@ Sector size for .Nm ggate provider. If not specified, it is taken from the device, or set to 512 bytes for files. +.It Fl T Ar remote_host:port +Use SOCK5 to open connection to remote_host:port before switching +to the ggated protocol. .It Fl t Ar timeout Number of seconds to wait before an I/O request will be canceled. Default is 0, which means no timeout. @@ -167,6 +172,14 @@ server# ggated client# ggatec create -o ro server /dev/acd0 ggate0 client# mount_cd9660 /dev/ggate0 /cdrom + +.Ed +Connect to 127.0.1.1:9050, SOCKS5-negotiate a connection to +the Tor location hidden service czdqtfrgvizltdal.onion:1312 +and access a ZVOL: +.Bd -literal -offset indent +# ggatec create -T czdqtfrgvizltdal.onion:1312 -p 9050 \\ + 127.0.1.1 /dev/zvol/dpool/ggated/czdqtfrgvizltdal.eli .Ed .Sh SEE ALSO .Xr geom 4 , -- 2.11.0 From 0592a79d37cbb7cc0589000d6aa8e35cb9e6ed85 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:10:17 +0200 Subject: [PATCH 027/257] ggatec: Reject unexpected GGATE commands in recv_thread() Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ed5b94915ac6..67a0f405047e 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -220,6 +220,11 @@ recv_thread(void *arg __unused) ggio.gctl_length = hdr.gh_length; ggio.gctl_error = hdr.gh_error; + if (ggio.gctl_cmd != GGATE_CMD_READ && + ggio.gctl_cmd != GGATE_CMD_WRITE) { + g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); + } + if (ggio.gctl_error == 0 && ggio.gctl_cmd == GGATE_CMD_READ) { data = g_gate_recv(recvfd, ggio.gctl_data, ggio.gctl_length, MSG_WAITALL); -- 2.11.0 From 87a7b31db2f8ad7ca3251a2045f71e9baac0ce9e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:15:18 +0200 Subject: [PATCH 028/257] ggatec: Log if the remote side signals errors Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 67a0f405047e..dd75c6468e9f 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -225,6 +225,12 @@ recv_thread(void *arg __unused) g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); } + if (ggio.gctl_error != 0) { + g_gate_log(LOG_ERR, + "Remote side signaled error %d: %s.", + ggio.gctl_error, strerror(ggio.gctl_error)); + } + if (ggio.gctl_error == 0 && ggio.gctl_cmd == GGATE_CMD_READ) { data = g_gate_recv(recvfd, ggio.gctl_data, ggio.gctl_length, MSG_WAITALL); -- 2.11.0 From c1d034958dfd0431d8020bd8fac24a28e8141e61 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:39:25 +0200 Subject: [PATCH 029/257] ggatec: Reject unsupported BIO commands Due to missing sanity checks in ggatec, requests like BIO_FLUSH and BIO_DELETE were previously accepted and sent as zero-size reads or writes. Also due to missing sanity checks, ggated did not mind. Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index dd75c6468e9f..650e4f585d54 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -146,6 +146,13 @@ send_thread(void *arg __unused) case BIO_WRITE: hdr.gh_cmd = GGATE_CMD_WRITE; break; + default: + g_gate_log(LOG_ERR, + "Rejecting unsupported BIO command: %d", + ggio.gctl_cmd); + ggio.gctl_error = EOPNOTSUPP; + g_gate_ioctl(G_GATE_CMD_DONE, &ggio); + continue; } hdr.gh_seq = ggio.gctl_seq; hdr.gh_offset = ggio.gctl_offset; -- 2.11.0 From 01345021a4a9800f1248bd37036ccd5edfb009c9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 27 Apr 2015 19:53:30 +0200 Subject: [PATCH 030/257] ggate[cd]: Add BIO_FLUSH support Let ggated transform BIO_FLUSH requests into fsync() calls. Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 8 +++++++- sbin/ggate/ggated/ggated.c | 23 ++++++++++++++++++----- sbin/ggate/shared/ggate.h | 1 + 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 650e4f585d54..f91c172effb2 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -146,6 +146,11 @@ send_thread(void *arg __unused) case BIO_WRITE: hdr.gh_cmd = GGATE_CMD_WRITE; break; + case BIO_FLUSH: + g_gate_log(LOG_DEBUG, "FLUSH request"); + hdr.gh_cmd = GGATE_CMD_FLUSH; + assert(ggio.gctl_length == 0); + break; default: g_gate_log(LOG_ERR, "Rejecting unsupported BIO command: %d", @@ -228,7 +233,8 @@ recv_thread(void *arg __unused) ggio.gctl_error = hdr.gh_error; if (ggio.gctl_cmd != GGATE_CMD_READ && - ggio.gctl_cmd != GGATE_CMD_WRITE) { + ggio.gctl_cmd != GGATE_CMD_WRITE && + ggio.gctl_cmd != GGATE_CMD_FLUSH) { g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); } diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 22810b0a00fd..2adbb68f7a83 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -657,8 +657,10 @@ recv_thread(void *arg) * Reject requests that violate assertions in disk_thread(). */ if (req->r_cmd != GGATE_CMD_READ && - req->r_cmd != GGATE_CMD_WRITE) { - g_gate_xlog("Request contains invalid command."); + req->r_cmd != GGATE_CMD_WRITE && + req->r_cmd != GGATE_CMD_FLUSH) { + g_gate_xlog("Request contains invalid command: %d", + req->r_cmd); } if (req->r_offset + req->r_length > (uintmax_t)conn->c_mediasize) { @@ -681,9 +683,10 @@ recv_thread(void *arg) } /* - * Allocate memory for data. + * Allocate memory for data, except when flushing. */ - req->r_data = malloc_waitok(req->r_length); + req->r_data = req->r_cmd != GGATE_CMD_FLUSH ? + malloc_waitok(req->r_length) : NULL; /* * Receive data to write for WRITE request. @@ -743,7 +746,9 @@ disk_thread(void *arg) /* * Check the request. */ - assert(req->r_cmd == GGATE_CMD_READ || req->r_cmd == GGATE_CMD_WRITE); + assert(req->r_cmd == GGATE_CMD_READ || + req->r_cmd == GGATE_CMD_WRITE || + req->r_cmd == GGATE_CMD_FLUSH); assert(req->r_offset + req->r_length <= (uintmax_t)conn->c_mediasize); assert((req->r_offset % conn->c_sectorsize) == 0); assert((req->r_length % conn->c_sectorsize) == 0); @@ -767,6 +772,14 @@ disk_thread(void *arg) free(req->r_data); req->r_data = NULL; break; + case GGATE_CMD_FLUSH: + g_gate_log(LOG_DEBUG, "Flushing"); + if (fsync(fd)) { + req->r_error = errno; + g_gate_log(LOG_ERR, "Flushing failed: %s", + strerror(errno)); + } + break; } if (data != (ssize_t)req->r_length) { /* Report short reads/writes as I/O errors. */ diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 898efea6507e..52985660def6 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -55,6 +55,7 @@ #define GGATE_CMD_READ 0 #define GGATE_CMD_WRITE 1 +#define GGATE_CMD_FLUSH 2 extern int g_gate_devfd; extern int g_gate_verbose; -- 2.11.0 From 8bf08d23e3b88fed0b1d0c4b56727439783d4a6f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 29 Apr 2015 12:44:56 +0200 Subject: [PATCH 031/257] ggatec: Log the command type for hdr packets (when debugging) ... and provide more details about failed requests. Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 11 ++++++++--- sbin/ggate/shared/ggate.c | 16 ++++++++++++++++ sbin/ggate/shared/ggate.h | 1 + 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index f91c172effb2..4d0b12f940ee 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -166,7 +166,9 @@ send_thread(void *arg __unused) g_gate_swap2n_hdr(&hdr); data = g_gate_send(sendfd, &hdr, sizeof(hdr), MSG_NOSIGNAL); - g_gate_log(LOG_DEBUG, "Sent hdr packet."); + g_gate_log(LOG_DEBUG, "Sent hdr packet (%s).", + g_gate_cmd2str(hdr.gh_cmd)); + g_gate_swap2h_hdr(&hdr); if (reconnect) break; @@ -224,7 +226,8 @@ recv_thread(void *arg __unused) pthread_kill(sendtd, SIGUSR1); break; } - g_gate_log(LOG_DEBUG, "Received hdr packet."); + g_gate_log(LOG_DEBUG, "Received hdr packet (%s).", + g_gate_cmd2str(hdr.gh_cmd)); ggio.gctl_seq = hdr.gh_seq; ggio.gctl_cmd = hdr.gh_cmd; @@ -240,7 +243,9 @@ recv_thread(void *arg __unused) if (ggio.gctl_error != 0) { g_gate_log(LOG_ERR, - "Remote side signaled error %d: %s.", + "%s for %d bytes at offset %d failed. " + "Error %d: %s.", g_gate_cmd2str(ggio.gctl_cmd), + ggio.gctl_length, ggio.gctl_offset, ggio.gctl_error, strerror(ggio.gctl_error)); } diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index cf9b9ca1574f..05fc25a9afc0 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -407,3 +407,19 @@ g_gate_str2ip(const char *str) return (INADDR_NONE); return (((struct in_addr *)(void *)hp->h_addr)->s_addr); } + +const char * +g_gate_cmd2str(int cmd) +{ + + switch (cmd) { + case GGATE_CMD_READ: + return ("GGATE_CMD_READ"); + case GGATE_CMD_WRITE: + return ("GGATE_CMD_WRITE"); + case GGATE_CMD_FLUSH: + return ("GGATE_CMD_FLUSH"); + } + + return ("unknown (invalid?) GGATE command"); +} diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 52985660def6..36565192a1a8 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -112,6 +112,7 @@ void g_gate_socket_settings(int sfd); void g_gate_list(int unit, int verbose); #endif in_addr_t g_gate_str2ip(const char *str); +const char *g_gate_cmd2str(int cmd); /* * g_gate_swap2h_* - functions swap bytes to host byte order (from big endian). -- 2.11.0 From 8c3552365c4fc2245541604aec83e56d01c880a5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 6 May 2015 15:55:08 +0200 Subject: [PATCH 032/257] ggated disk_thread(): Include the command in the debug output Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 2adbb68f7a83..fa11d8b90d13 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -753,7 +753,8 @@ disk_thread(void *arg) assert((req->r_offset % conn->c_sectorsize) == 0); assert((req->r_length % conn->c_sectorsize) == 0); - g_gate_log(LOG_DEBUG, "%s: offset=%jd length=%u", __func__, + g_gate_log(LOG_DEBUG, "%s: cmd=%s offset=%jd length=%u", + __func__, g_gate_cmd2str(req->r_cmd), (intmax_t)req->r_offset, (unsigned)req->r_length); /* -- 2.11.0 From c834a7854be580c3b63d512b72eadcc8fd948c3e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 5 May 2015 17:39:16 +0200 Subject: [PATCH 033/257] ggate[cd]: Add BIO_DELETE support On the ggated side the requests are translated into writes of zero which ZFS will convert into BIO_DELETE requests again when zle compression is enabled. Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 5 +++++ sbin/ggate/ggated/ggated.c | 47 +++++++++++++++++++++++++++++++++++++++------- sbin/ggate/shared/ggate.c | 2 ++ sbin/ggate/shared/ggate.h | 1 + 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index 4d0b12f940ee..e70cf553f359 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -146,6 +146,10 @@ send_thread(void *arg __unused) case BIO_WRITE: hdr.gh_cmd = GGATE_CMD_WRITE; break; + case BIO_DELETE: + g_gate_log(LOG_DEBUG, "DELETE request"); + hdr.gh_cmd = GGATE_CMD_DELETE; + break; case BIO_FLUSH: g_gate_log(LOG_DEBUG, "FLUSH request"); hdr.gh_cmd = GGATE_CMD_FLUSH; @@ -237,6 +241,7 @@ recv_thread(void *arg __unused) if (ggio.gctl_cmd != GGATE_CMD_READ && ggio.gctl_cmd != GGATE_CMD_WRITE && + ggio.gctl_cmd != GGATE_CMD_DELETE && ggio.gctl_cmd != GGATE_CMD_FLUSH) { g_gate_xlog("Unexpected GGATE_CMD: %d", ggio.gctl_cmd); } diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index fa11d8b90d13..c640b5798f69 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -636,6 +636,7 @@ recv_thread(void *arg) * Get header packet. */ req = malloc_waitok(sizeof(*req)); + memset(req, 0, sizeof(*req)); data = g_gate_recv(fd, &req->r_hdr, sizeof(req->r_hdr), MSG_WAITALL); if (data == 0) { @@ -658,6 +659,7 @@ recv_thread(void *arg) */ if (req->r_cmd != GGATE_CMD_READ && req->r_cmd != GGATE_CMD_WRITE && + req->r_cmd != GGATE_CMD_DELETE && req->r_cmd != GGATE_CMD_FLUSH) { g_gate_xlog("Request contains invalid command: %d", req->r_cmd); @@ -677,21 +679,16 @@ recv_thread(void *arg) * the client. MAXPHYS is the hard limit in ggatec, * values above it are thus pretty suspicious. */ - if (req->r_length > MAXPHYS) { + if (req->r_length > MAXPHYS && req->r_cmd != GGATE_CMD_DELETE) { g_gate_xlog("Request length above MAXPHYS: %u > %u", (unsigned)req->r_length, MAXPHYS); } /* - * Allocate memory for data, except when flushing. - */ - req->r_data = req->r_cmd != GGATE_CMD_FLUSH ? - malloc_waitok(req->r_length) : NULL; - - /* * Receive data to write for WRITE request. */ if (req->r_cmd == GGATE_CMD_WRITE) { + req->r_data = malloc_waitok(req->r_length); g_gate_log(LOG_DEBUG, "Waiting for %u bytes of data...", req->r_length); data = g_gate_recv(fd, req->r_data, req->r_length, @@ -718,6 +715,34 @@ recv_thread(void *arg) } } +static ssize_t +delete_range(int fd, size_t length, off_t offset) +{ + static char zeros[MAXPHYS]; + size_t written; + + written = 0; + + do + { + int ret; + size_t bytes_left; + size_t chunk_size; + + bytes_left = length - written; + chunk_size = bytes_left > MAXPHYS ? MAXPHYS : bytes_left; + ret = pwrite(fd, zeros, chunk_size, offset + written); + if (ret == -1) + return (written); + written += ret; + } while (written < length); + + g_gate_log(LOG_DEBUG, "Overwritten %u bytes at offset %jd with zeros", + written, (intmax_t)offset); + + return (written); +} + static void * disk_thread(void *arg) { @@ -748,6 +773,7 @@ disk_thread(void *arg) */ assert(req->r_cmd == GGATE_CMD_READ || req->r_cmd == GGATE_CMD_WRITE || + req->r_cmd == GGATE_CMD_DELETE || req->r_cmd == GGATE_CMD_FLUSH); assert(req->r_offset + req->r_length <= (uintmax_t)conn->c_mediasize); assert((req->r_offset % conn->c_sectorsize) == 0); @@ -763,9 +789,16 @@ disk_thread(void *arg) data = 0; switch (req->r_cmd) { case GGATE_CMD_READ: + assert(req->r_data == NULL); + req->r_data = malloc_waitok(req->r_length); data = pread(fd, req->r_data, req->r_length, req->r_offset); break; + case GGATE_CMD_DELETE: + data = delete_range(fd, req->r_length, + req->r_offset); + assert((size_t)data <= req->r_length); + break; case GGATE_CMD_WRITE: data = pwrite(fd, req->r_data, req->r_length, req->r_offset); diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index 05fc25a9afc0..7fe37cc0bc91 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -417,6 +417,8 @@ g_gate_cmd2str(int cmd) return ("GGATE_CMD_READ"); case GGATE_CMD_WRITE: return ("GGATE_CMD_WRITE"); + case GGATE_CMD_DELETE: + return ("GGATE_CMD_DELETE"); case GGATE_CMD_FLUSH: return ("GGATE_CMD_FLUSH"); } diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 36565192a1a8..e5792e3c41e3 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -56,6 +56,7 @@ #define GGATE_CMD_READ 0 #define GGATE_CMD_WRITE 1 #define GGATE_CMD_FLUSH 2 +#define GGATE_CMD_DELETE 3 extern int g_gate_devfd; extern int g_gate_verbose; -- 2.11.0 From 3b63e0c03dc3b05a758e8943b177cbe40e9825a1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 29 Apr 2015 10:55:40 +0200 Subject: [PATCH 034/257] ggated send_thread(): Assert that we only send data for read requests Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index c640b5798f69..8e564e136658 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -883,6 +883,7 @@ send_thread(void *arg) g_gate_log(LOG_DEBUG, "Sent hdr packet."); g_gate_swap2h_hdr(&req->r_hdr); if (req->r_data != NULL) { + assert(req->r_cmd == GGATE_CMD_READ); data = g_gate_send(fd, req->r_data, req->r_length, 0); if (data != (ssize_t)req->r_length) { g_gate_xlog("Error while sending data: %s.", -- 2.11.0 From a2b84456c1068cd394ec3de105d363650eab2d1e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 May 2015 18:00:04 +0200 Subject: [PATCH 035/257] ggated: Open the listening socket CLOEXEC Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 8e564e136658..49e9c58d67df 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1109,7 +1109,7 @@ main(int argc, char *argv[]) signal(SIGCHLD, SIG_IGN); signal(SIGPIPE, SIG_IGN); - sfd = socket(AF_INET, SOCK_STREAM, 0); + sfd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (sfd == -1) g_gate_xlog("Cannot open stream socket: %s.", strerror(errno)); bzero(&serv, sizeof(serv)); -- 2.11.0 From 6ee2f387fce4a58958e1a0b1fb06c0b152706386 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 May 2015 18:31:46 +0200 Subject: [PATCH 036/257] ggated: Fix another socket leak Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 49e9c58d67df..383386b57cef 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -341,6 +341,11 @@ exports_check(struct ggd_export *ex, struct g_gate_cinit *cinit, return (EPERM); } } + if (conn->c_diskfd != -1) { + g_gate_log(LOG_DEBUG, "Requested file %s is already open: %d", + ex->e_path, conn->c_diskfd); + return(0); + } if ((conn->c_flags & GGATE_FLAG_RDONLY) != 0) flags = O_RDONLY; else if ((conn->c_flags & GGATE_FLAG_WRONLY) != 0) -- 2.11.0 From 5b0474804c95b1d4f3b02a5b2a41c436ae017433 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 30 Apr 2015 11:52:06 +0200 Subject: [PATCH 037/257] ggated recv_thread(): In case of read-only files, only accept read commands Accepting write commands etc. is not a security problem because the file descriptor isn't writeable anyway, but accepting requests other than reads could hide client bugs. Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 383386b57cef..5ae9d2f7a8f1 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -679,6 +679,12 @@ recv_thread(void *arg) "not fit sector size."); } + if ((conn->c_flags & GGATE_FLAG_RDONLY) != 0 + && req->r_cmd != GGATE_CMD_READ) { + g_gate_xlog("%s request received for read-only file", + g_gate_cmd2str(req->r_cmd)); + } + /* * Limit the amount of memory we allocate on behalf of * the client. MAXPHYS is the hard limit in ggatec, -- 2.11.0 From 7d881ad62af8b2ab17af52176415024d580aa34e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 3 May 2015 14:02:02 +0200 Subject: [PATCH 038/257] ggatec: Add log-to-file support Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.c | 5 ++++- sbin/ggate/shared/ggate.c | 26 +++++++++++++++++++++----- sbin/ggate/shared/ggate.h | 1 + 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index e70cf553f359..ba68977cbded 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -616,7 +616,7 @@ main(int argc, char *argv[]) int ch; char *p; - ch = getopt(argc, argv, "fno:p:q:R:S:s:t:T:u:v"); + ch = getopt(argc, argv, "fl:no:p:q:R:S:s:t:T:u:v"); if (ch == -1) break; switch (ch) { @@ -625,6 +625,9 @@ main(int argc, char *argv[]) usage(); force = 1; break; + case 'l': + g_gate_open_log(optarg); + break; case 'n': if (action != CREATE && action != RESCUE) usage(); diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index 7fe37cc0bc91..b5799d6c1620 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -26,6 +26,7 @@ * $FreeBSD$ */ +#define _WITH_DPRINTF #include #include #include @@ -59,13 +60,23 @@ int g_gate_devfd = -1; int g_gate_verbose = 0; +static int g_gate_logfd = -1; +void +g_gate_open_log(const char *logfile) +{ + + g_gate_logfd = open(logfile, O_CREAT | O_WRONLY | O_APPEND, S_IWUSR |S_IRUSR); + if (g_gate_logfd == -1) { + g_gate_xlog("Failed to open %s: %s", logfile, strerror(errno)); + } +} void g_gate_vlog(int priority, const char *message, va_list ap) { - if (g_gate_verbose) { + if (g_gate_verbose || g_gate_logfd != -1) { const char *prefix; switch (priority) { @@ -87,10 +98,15 @@ g_gate_vlog(int priority, const char *message, va_list ap) default: prefix = "unknown"; } - - printf("%s: ", prefix); - vprintf(message, ap); - printf("\n"); + if (g_gate_logfd == -1) { + printf("%s: ", prefix); + vprintf(message, ap); + printf("\n"); + } else if (g_gate_verbose || priority != LOG_DEBUG) { + dprintf(g_gate_logfd, "%s: ", prefix); + vdprintf(g_gate_logfd, message, ap); + dprintf(g_gate_logfd, "\n"); + } } else { if (priority != LOG_DEBUG) vsyslog(priority, message, ap); diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index e5792e3c41e3..51f425dc8cbf 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -95,6 +95,7 @@ struct g_gate_hdr { uint16_t gh_error; /* error value (0 if ok) */ } __packed; +void g_gate_open_log(const char *logfile); void g_gate_vlog(int priority, const char *message, va_list ap); void g_gate_log(int priority, const char *message, ...); void g_gate_xvlog(const char *message, va_list ap) __dead2; -- 2.11.0 From 1bde5c051a2fe70eaca609279aca3f6046b7020e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 30 Apr 2015 13:52:39 +0200 Subject: [PATCH 039/257] ggate[cd]: Add Jail and Capsicum support The capsicum support for ggatec is incomplete and only enabled if the -c flag is used as it currently prevents ggatec from reconnecting which is very inconvenient. Obtained from: ElectroBSD --- sbin/ggate/ggatec/ggatec.8 | 8 +++ sbin/ggate/ggatec/ggatec.c | 37 +++++++++--- sbin/ggate/ggated/ggated.c | 5 ++ sbin/ggate/shared/ggate.c | 137 +++++++++++++++++++++++++++++++++++++++++++++ sbin/ggate/shared/ggate.h | 2 + 5 files changed, 182 insertions(+), 7 deletions(-) diff --git a/sbin/ggate/ggatec/ggatec.8 b/sbin/ggate/ggatec/ggatec.8 index c50416f4e418..57b6d428efff 100644 --- a/sbin/ggate/ggatec/ggatec.8 +++ b/sbin/ggate/ggatec/ggatec.8 @@ -33,6 +33,7 @@ .Sh SYNOPSIS .Nm .Cm create +.Op Fl c .Op Fl n .Op Fl v .Op Fl o Cm ro | wo | rw @@ -48,6 +49,7 @@ .Ar path .Nm .Cm rescue +.Op Fl c .Op Fl n .Op Fl v .Op Fl o Cm ro | wo | rw @@ -104,6 +106,12 @@ providers. .Pp Available options: .Bl -tag -width ".Fl s Cm ro | wo | rw" +.It Fl c +Enter capsicum sandbox. +Currently this prevents +.Nm ggatec +from reconnecting which is somewhat inconvenient. +The flag will go away once this is fixed. .It Fl f Forcibly destroy .Nm ggate diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ba68977cbded..ad2db8836891 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -50,6 +50,7 @@ #include #include #include +#include #include #include "ggate.h" @@ -59,6 +60,8 @@ static enum { UNSET, CREATE, DESTROY, LIST, RESCUE } action = UNSET; static const char *path = NULL; static const char *host = NULL; +static in_addr_t host_ip; +static const char *logfile = NULL; static int unit = G_GATE_UNIT_AUTO; static unsigned flags = 0; static int force = 0; @@ -73,6 +76,7 @@ static int sendfd, recvfd; static uint32_t token; static pthread_t sendtd, recvtd; static int reconnect; +static int drop_capabilities = 0; static void usage(void) @@ -353,7 +357,7 @@ handshake(int dir) */ bzero(&serv, sizeof(serv)); serv.sin_family = AF_INET; - serv.sin_addr.s_addr = g_gate_str2ip(host); + serv.sin_addr.s_addr = host_ip; if (serv.sin_addr.s_addr == INADDR_NONE) { g_gate_log(LOG_DEBUG, "Invalid IP/host name: %s.", host); return (-1); @@ -461,7 +465,7 @@ static void mydaemon(void) { - if (g_gate_verbose > 0) + if (logfile == NULL && g_gate_verbose > 0) return; if (daemon(0, 0) == 0) return; @@ -526,6 +530,10 @@ g_gatec_loop(void) signal(SIGUSR1, signop); for (;;) { g_gatec_start(); + + if (cap_sandboxed()) + g_gate_xlog("Got disconnected while being sandboxed."); + g_gate_log(LOG_NOTICE, "Disconnected [%s %s]. Connecting...", host, path); while (!g_gatec_connect()) { @@ -548,9 +556,6 @@ g_gatec_create(void) if (!g_gatec_connect()) g_gate_xlog("Cannot connect: %s.", strerror(errno)); - /* - * Ok, got both sockets, time to create provider. - */ memset(&ggioc, 0, sizeof(ggioc)); ggioc.gctl_version = G_GATE_VERSION; ggioc.gctl_mediasize = mediasize; @@ -573,6 +578,9 @@ g_gatec_create(void) } unit = ggioc.gctl_unit; + if (drop_capabilities) + g_gate_drop_capabilities(sendfd, recvfd); + mydaemon(); g_gatec_loop(); } @@ -585,6 +593,9 @@ g_gatec_rescue(void) if (!g_gatec_connect()) g_gate_xlog("Cannot connect: %s.", strerror(errno)); + if (drop_capabilities) + g_gate_drop_capabilities(sendfd, recvfd); + ggioc.gctl_version = G_GATE_VERSION; ggioc.gctl_unit = unit; ggioc.gctl_seq = 0; @@ -616,17 +627,21 @@ main(int argc, char *argv[]) int ch; char *p; - ch = getopt(argc, argv, "fl:no:p:q:R:S:s:t:T:u:v"); + ch = getopt(argc, argv, "cfl:no:p:q:R:S:s:t:T:u:v"); if (ch == -1) break; switch (ch) { + case 'c': + drop_capabilities = 1; + force = 1; + break; case 'f': if (action != DESTROY) usage(); force = 1; break; case 'l': - g_gate_open_log(optarg); + logfile = optarg; break; case 'n': if (action != CREATE && action != RESCUE) @@ -738,7 +753,11 @@ main(int argc, char *argv[]) g_gate_load_module(); g_gate_open_device(); host = argv[0]; + host_ip = g_gate_str2ip(host); path = argv[1]; + if (logfile != NULL) + g_gate_open_log(logfile); + g_gate_drop_privs("hast", host_ip); g_gatec_create(); break; case DESTROY: @@ -762,7 +781,11 @@ main(int argc, char *argv[]) } g_gate_open_device(); host = argv[0]; + host_ip = g_gate_str2ip(host); path = argv[1]; + if (logfile != NULL) + g_gate_open_log(logfile); + g_gate_drop_privs("hast", host_ip); g_gatec_rescue(); break; case UNSET: diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 5ae9d2f7a8f1..e7dbfbff36b8 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -545,6 +545,11 @@ connection_launch(struct ggd_connection *conn) } g_gate_log(LOG_DEBUG, "Process created [%s].", conn->c_path); + if (getuid() == 0) + g_gate_drop_privs("hast", bindaddr); + + g_gate_drop_capabilities(conn->c_sendfd, conn->c_recvfd); + /* * Create condition variables and mutexes for in-queue and out-queue * synchronization. diff --git a/sbin/ggate/shared/ggate.c b/sbin/ggate/shared/ggate.c index b5799d6c1620..13277eae4a43 100644 --- a/sbin/ggate/shared/ggate.c +++ b/sbin/ggate/shared/ggate.c @@ -53,6 +53,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include "ggate.h" @@ -441,3 +446,135 @@ g_gate_cmd2str(int cmd) return ("unknown (invalid?) GGATE command"); } + +/* + * The functions below are based on drop_privs() from ../../hastd/subr.c + * + * Changes: + * - HAST_USER replaced with ggate_user option + * - pjdlog_* replaced with g_gate_xlog(). + * - Don't fall back to chroot if jailing fails. + */ +#define PJDLOG_VERIFY assert +void +g_gate_drop_privs(const char *ggate_user, in_addr_t jail_address) +{ + char jailhost[32]; + struct jail jailst; + struct passwd *pw; + uid_t ruid, euid, suid; + gid_t rgid, egid, sgid; + gid_t gidset[1]; + struct in_addr jail_ip; + /* + * According to getpwnam(3) we have to clear errno before calling the + * function to be able to distinguish between an error and missing + * entry (with is not treated as error by getpwnam(3)). + */ + errno = 0; + pw = getpwnam(ggate_user); + if (pw == NULL) { + if (errno != 0) { + g_gate_xlog("Unable to find info about '%s' user", + ggate_user); + } else { + g_gate_xlog("'%s' user doesn't exist.", ggate_user); + } + } + + jail_ip.s_addr = jail_address; + + bzero(&jailst, sizeof(jailst)); + jailst.version = JAIL_API_VERSION; + jailst.path = pw->pw_dir; + (void)snprintf(jailhost, sizeof(jailhost), "%s-jail", getprogname()); + jailst.hostname = jailhost; + jailst.jailname = NULL; + jailst.ip4s = 1; + jailst.ip4 = &jail_ip; + jailst.ip6s = 0; + jailst.ip6 = NULL; + if (jail(&jailst) == -1) { + g_gate_xlog("Unable to jail process in directory %s", pw->pw_dir); + } + PJDLOG_VERIFY(chdir("/") == 0); + gidset[0] = pw->pw_gid; + if (setgroups(1, gidset) == -1) { + g_gate_xlog("Unable to set groups to gid %u", + (unsigned int)pw->pw_gid); + } + if (setgid(pw->pw_gid) == -1) { + g_gate_xlog("Unable to set gid to %u", + (unsigned int)pw->pw_gid); + } + if (setuid(pw->pw_uid) == -1) { + g_gate_xlog("Unable to set uid to %u", + (unsigned int)pw->pw_uid); + } + + /* + * Better be sure that everything succeeded. + */ + PJDLOG_VERIFY(getresuid(&ruid, &euid, &suid) == 0); + PJDLOG_VERIFY(ruid == pw->pw_uid); + PJDLOG_VERIFY(euid == pw->pw_uid); + PJDLOG_VERIFY(suid == pw->pw_uid); + PJDLOG_VERIFY(getresgid(&rgid, &egid, &sgid) == 0); + PJDLOG_VERIFY(rgid == pw->pw_gid); + PJDLOG_VERIFY(egid == pw->pw_gid); + PJDLOG_VERIFY(sgid == pw->pw_gid); + PJDLOG_VERIFY(getgroups(0, NULL) == 1); + PJDLOG_VERIFY(getgroups(1, gidset) == 1); + PJDLOG_VERIFY(gidset[0] == pw->pw_gid); + + g_gate_log(LOG_DEBUG, "Privileges successfully dropped using " + "jail+setgid+setuid."); +} + +int +g_gate_drop_capabilities(int sendfd, int recvfd) +{ + cap_rights_t rights; + static const unsigned long ggatecmds[] = { + G_GATE_CMD_START, + G_GATE_CMD_DONE, + G_GATE_CMD_CANCEL, + }; + + if (cap_enter() != 0) { + g_gate_xlog("Failed to sandbox using capsicum"); + } + + cap_rights_init(&rights, CAP_PREAD, CAP_PWRITE); + if (cap_rights_limit(sendfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability " + "rights on sendfd %d", sendfd); + } + if (cap_rights_limit(recvfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability " + "rights on recvfd %d", recvfd); + } + + /* Only the client uses this. */ + if (g_gate_devfd != -1) { + cap_rights_init(&rights, CAP_IOCTL, CAP_PREAD, CAP_PWRITE); + if (cap_rights_limit(g_gate_devfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability rights " + "to CAP_IOCTL on ggate descriptor"); + } + if (cap_ioctls_limit(g_gate_devfd, ggatecmds, + sizeof(ggatecmds) / sizeof(ggatecmds[0])) == -1) { + g_gate_xlog("Unable to limit allowed ggate ioctls"); + } + } + cap_rights_init(&rights, CAP_PWRITE); + if (g_gate_logfd != -1 && + cap_rights_limit(g_gate_logfd, &rights) == -1) { + g_gate_xlog("Unable to limit capability " + "rights on logfd %d", g_gate_logfd); + } + + g_gate_log(LOG_DEBUG, "Entered Capsicum sandbox"); + + return (0); +} diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index 51f425dc8cbf..cdc3cbfdbb64 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -115,6 +115,8 @@ void g_gate_list(int unit, int verbose); #endif in_addr_t g_gate_str2ip(const char *str); const char *g_gate_cmd2str(int cmd); +void g_gate_drop_privs(const char *ggate_user, in_addr_t jail_address); +int g_gate_drop_capabilities(int sendfd, int recvfd); /* * g_gate_swap2h_* - functions swap bytes to host byte order (from big endian). -- 2.11.0 From ba13621a6f87a862011c0dad1cabb01bf0349efd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 28 Apr 2015 13:02:25 +0200 Subject: [PATCH 040/257] Bump GGATE_VERSION due to FLUSH and DELETE support and various bug fixes Unpatched ggate[cd] versions may cause data corruption so we no longer want to speak to them. Obtained from: ElectroBSD --- sbin/ggate/shared/ggate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/shared/ggate.h b/sbin/ggate/shared/ggate.h index cdc3cbfdbb64..e764665ebb85 100644 --- a/sbin/ggate/shared/ggate.h +++ b/sbin/ggate/shared/ggate.h @@ -40,7 +40,7 @@ #define G_GATE_TIMEOUT 0 #define GGATE_MAGIC "GEOM_GATE " -#define GGATE_VERSION 0 +#define GGATE_VERSION 1 #define GGATE_FLAG_RDONLY 0x0001 #define GGATE_FLAG_WRONLY 0x0002 -- 2.11.0 From ef018e08602f65df26c9fc86012f0525fc1a88f3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 9 Aug 2015 15:20:48 +0200 Subject: [PATCH 041/257] Use dedicated users for ggatec and ggated Obtained from: ElectroBSD --- etc/group | 2 ++ etc/master.passwd | 2 ++ sbin/ggate/ggatec/ggatec.c | 2 +- sbin/ggate/ggated/ggated.c | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/group b/etc/group index c20f1d93dd3c..3136230b0790 100644 --- a/etc/group +++ b/etc/group @@ -31,5 +31,7 @@ audit:*:77: www:*:80: _ypldap:*:160: hast:*:845: +ggatec:*:846: +ggated:*:847: nogroup:*:65533: nobody:*:65534: diff --git a/etc/master.passwd b/etc/master.passwd index 4b0f3645cd3d..689b64e78473 100644 --- a/etc/master.passwd +++ b/etc/master.passwd @@ -24,4 +24,6 @@ auditdistd:*:78:77::0:0:Auditdistd unprivileged user:/var/empty:/usr/sbin/nologi www:*:80:80::0:0:World Wide Web Owner:/nonexistent:/usr/sbin/nologin _ypldap:*:160:160::0:0:YP LDAP unprivileged user:/var/empty:/usr/sbin/nologin hast:*:845:845::0:0:HAST unprivileged user:/var/empty:/usr/sbin/nologin +ggatec:*:846:846::0:0:ggatec unprivileged user:/var/empty:/usr/sbin/nologin +ggated:*:847:847::0:0:ggated unprivileged user:/var/empty:/usr/sbin/nologin nobody:*:65534:65534::0:0:Unprivileged user:/nonexistent:/usr/sbin/nologin diff --git a/sbin/ggate/ggatec/ggatec.c b/sbin/ggate/ggatec/ggatec.c index ad2db8836891..cd33e1238ac5 100644 --- a/sbin/ggate/ggatec/ggatec.c +++ b/sbin/ggate/ggatec/ggatec.c @@ -757,7 +757,7 @@ main(int argc, char *argv[]) path = argv[1]; if (logfile != NULL) g_gate_open_log(logfile); - g_gate_drop_privs("hast", host_ip); + g_gate_drop_privs("ggatec", host_ip); g_gatec_create(); break; case DESTROY: diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index e7dbfbff36b8..439725314aab 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -546,7 +546,7 @@ connection_launch(struct ggd_connection *conn) g_gate_log(LOG_DEBUG, "Process created [%s].", conn->c_path); if (getuid() == 0) - g_gate_drop_privs("hast", bindaddr); + g_gate_drop_privs("ggated", bindaddr); g_gate_drop_capabilities(conn->c_sendfd, conn->c_recvfd); -- 2.11.0 From 6fb891709ad5cd03a0017ff68e2fdce13b337fc9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Jan 2015 18:50:18 +0100 Subject: [PATCH 042/257] bge(4): Default to disallowing ASF It causes watchdog timeouts and undiagnosed permanent unresponsivenes on at least the 'CHIP ID 0x05784100; ASIC REV 0x5784; CHIP REV 0x57841;' in the evo-iv08 DL120 G6 I'm using for testing. I don't have time to debug the underlying cause right now and users who actually want ASF and have systems where it works can always enable it through loader.conf. Obtained from: ElectroBSD --- share/man/man4/bge.4 | 4 ++-- sys/dev/bge/if_bge.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/share/man/man4/bge.4 b/share/man/man4/bge.4 index 474b48fa5ff1..060d22d76658 100644 --- a/share/man/man4/bge.4 +++ b/share/man/man4/bge.4 @@ -31,7 +31,7 @@ .\" .\" $FreeBSD$ .\" -.Dd January 19, 2012 +.Dd January 19, 2015 .Dt BGE 4 .Os .Sh NAME @@ -196,7 +196,7 @@ prompt before booting the kernel, or stored in .It Va hw.bge.allow_asf Allow the ASF feature for cooperating with IPMI. Can cause system lockup problems on a small number of systems. -Enabled by default. +Disabled by default. .It Va dev.bge.%d.msi Non-zero value enables MSI support on the Ethernet hardware. The default value is 1. diff --git a/sys/dev/bge/if_bge.c b/sys/dev/bge/if_bge.c index 929bedc5ee0a..b89b8fb93af8 100644 --- a/sys/dev/bge/if_bge.c +++ b/sys/dev/bge/if_bge.c @@ -543,7 +543,7 @@ static devclass_t bge_devclass; DRIVER_MODULE(bge, pci, bge_driver, bge_devclass, 0, 0); DRIVER_MODULE(miibus, bge, miibus_driver, miibus_devclass, 0, 0); -static int bge_allow_asf = 1; +static int bge_allow_asf = 0; static SYSCTL_NODE(_hw, OID_AUTO, bge, CTLFLAG_RD, 0, "BGE driver parameters"); SYSCTL_INT(_hw_bge, OID_AUTO, allow_asf, CTLFLAG_RDTUN, &bge_allow_asf, 0, -- 2.11.0 From a82c10f85551e7b81aaced07428d33a796fd2b71 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Jan 2015 21:25:47 +0100 Subject: [PATCH 043/257] Assign random IP id values by default so users don't have to clown around with the sysctl themselves Obtained from: ElectroBSD --- sys/netinet/ip_id.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/netinet/ip_id.c b/sys/netinet/ip_id.c index 97d5851b349f..8651c6404730 100644 --- a/sys/netinet/ip_id.c +++ b/sys/netinet/ip_id.c @@ -137,8 +137,8 @@ static void ipid_sysuninit(void); SYSCTL_DECL(_net_inet_ip); SYSCTL_PROC(_net_inet_ip, OID_AUTO, random_id, CTLTYPE_INT | CTLFLAG_VNET | CTLFLAG_RW, - &VNET_NAME(ip_do_randomid), 0, sysctl_ip_randomid, "IU", - "Assign random ip_id values"); + &VNET_NAME(ip_do_randomid), 1, sysctl_ip_randomid, "IU", + "Assign random ip_id values. Important for Tor relays and a good idea in general."); SYSCTL_INT(_net_inet_ip, OID_AUTO, rfc6864, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip_rfc6864), 0, "Use constant IP ID for atomic datagrams"); -- 2.11.0 From fb972081877d27c9c36d2c25c2e204d978e4fbc8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 11 Feb 2015 12:19:24 +0100 Subject: [PATCH 044/257] Add vendor copyright ... after putting on my robe and "police educational technican" hat. This commit is optional. Feel free to import any other ElectroBSD commit without including this one. Obtained from: ElectroBSD --- sys/sys/copyright.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/sys/copyright.h b/sys/sys/copyright.h index e9c19881f284..938cbf02d374 100644 --- a/sys/sys/copyright.h +++ b/sys/sys/copyright.h @@ -30,7 +30,7 @@ /* Add a FreeBSD vendor copyright here */ #define COPYRIGHT_Vendor \ - "" + "Copyright (c) 2010-2017 Fabian Keil - IT-Beratung und Polizei-Erziehung\n" /* FreeBSD */ #define COPYRIGHT_FreeBSD \ -- 2.11.0 From e9367bccd7c7720c3a3dd892c530e8c228d1aeb2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 13 Feb 2015 19:37:33 +0100 Subject: [PATCH 045/257] OpenZFS: fix a comment typo Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c index 2a3939f4dec0..df53edad206c 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c @@ -1555,7 +1555,7 @@ int zfs_frag_table[FRAGMENTATION_TABLE_SIZE] = { }; /* - * Calclate the metaslab's fragmentation metric. A return value + * Calculate the metaslab's fragmentation metric. A return value * of ZFS_FRAG_INVALID means that the metaslab has not been upgraded and does * not support this metric. Otherwise, the return value should be in the * range [0, 100]. -- 2.11.0 From 114ece707b4a79272b65ae909e869177afb3d345 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 23 Feb 2015 11:38:09 +0100 Subject: [PATCH 046/257] newvers.sh: Set TYPE to ElectroBSD Obtained from: ElectroBSD --- sys/conf/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 sys/conf/newvers.sh diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh old mode 100644 new mode 100755 index fa05d7dea2c5..82b136426e6f --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -42,7 +42,7 @@ # checkout from a version control system. Metadata is # included if the tree is modified. -TYPE="FreeBSD" +TYPE="ElectroBSD" REVISION="11.0" BRANCH="STABLE" if [ -n "${BRANCH_OVERRIDE}" ]; then -- 2.11.0 From f33554915859ae71eb92c0cda4497b6460b74977 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 Jan 2016 13:45:32 +0100 Subject: [PATCH 047/257] mdocml: Change OS name used in man page headers to ElectroBSD Obtained from: ElectroBSD --- contrib/mdocml/msec.in | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/mdocml/msec.in b/contrib/mdocml/msec.in index 86d0dd8486f2..09f6f362c685 100644 --- a/contrib/mdocml/msec.in +++ b/contrib/mdocml/msec.in @@ -22,16 +22,16 @@ * Be sure to escape strings. */ -LINE("1", "FreeBSD General Commands Manual") -LINE("2", "FreeBSD System Calls Manual") -LINE("3", "FreeBSD Library Functions Manual") +LINE("1", "ElectroBSD General Commands Manual") +LINE("2", "ElectroBSD System Calls Manual") +LINE("3", "ElectroBSD Library Functions Manual") LINE("3p", "Perl Library Functions Manual") -LINE("4", "FreeBSD Kernel Interfaces Manual") -LINE("5", "FreeBSD File Formats Manual") -LINE("6", "FreeBSD Games Manual") -LINE("7", "FreeBSD Miscellaneous Information Manual") -LINE("8", "FreeBSD System Manager\'s Manual") -LINE("9", "FreeBSD Kernel Developer\'s Manual") +LINE("4", "ElectroBSD Kernel Interfaces Manual") +LINE("5", "ElectroBSD File Formats Manual") +LINE("6", "ElectroBSD Games Manual") +LINE("7", "ElectroBSD Miscellaneous Information Manual") +LINE("8", "ElectroBSD System Manager\'s Manual") +LINE("9", "ElectroBSD Kernel Developer\'s Manual") LINE("X11", "X11 Developer\'s Manual") LINE("X11R6", "X11 Developer\'s Manual") LINE("unass", "Unassociated") -- 2.11.0 From 859f4d496f344bda8eecd61b49e6ebed969fdd6c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 Jan 2016 20:24:00 +0100 Subject: [PATCH 048/257] clang: Set CLANG_VENDOR to ElectroBSD Obtained from: ElectroBSD --- lib/clang/include/clang/Basic/Version.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/clang/include/clang/Basic/Version.inc b/lib/clang/include/clang/Basic/Version.inc index b9d37e10f845..fa5700f89a58 100644 --- a/lib/clang/include/clang/Basic/Version.inc +++ b/lib/clang/include/clang/Basic/Version.inc @@ -5,6 +5,6 @@ #define CLANG_VERSION_MINOR 9 #define CLANG_VERSION_PATCHLEVEL 1 -#define CLANG_VENDOR "FreeBSD " +#define CLANG_VENDOR "ElectroBSD " #define SVN_REVISION "289601" -- 2.11.0 From 12728eb88e3e88f1dfb65a7d18b99bc6af3da056 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 24 Feb 2015 19:35:03 +0100 Subject: [PATCH 049/257] Let rc.d/motd work with unames other than FreeBSD Obtained from: ElectroBSD --- etc/rc.d/motd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/rc.d/motd b/etc/rc.d/motd index acb376723e80..11d2257aa9c0 100755 --- a/etc/rc.d/motd +++ b/etc/rc.d/motd @@ -35,7 +35,7 @@ motd_start() T=`mktemp -t motd` uname -v | sed -e 's,^\([^#]*\) #\(.* [1-2][0-9][0-9][0-9]\).*/\([^\]*\) $,\1 (\3) #\2,' > ${T} - awk '{if (NR == 1) {if ($1 == "FreeBSD") {next} else {print "\n"$0}} else {print}}' < /etc/motd >> ${T} + awk '{if (NR == 1) {if ($1 == "'"$(uname)"'") {next} else {print "\n"$0}} else {print}}' < /etc/motd >> ${T} cmp -s $T /etc/motd || { cp $T /etc/motd -- 2.11.0 From 957ad9473a1f5eee072380eafd2b6143b9a2c9ed Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 Mar 2015 12:07:50 +0100 Subject: [PATCH 050/257] swapon: Accept the "late" option for .eli devices Using the late option makes it possible to additionally use the consumer device as (unencrypted!) dump device without making it unavailable for rc.d/savecore (which runs between rc.d/swap and rc.d/swaplate). Obtained from: ElectroBSD Submitted in: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198598 Submitted at: 2015-03-15 11:37 UTC --- sbin/swapon/swapon.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbin/swapon/swapon.c b/sbin/swapon/swapon.c index 052fbc734afb..dfc4951c8137 100644 --- a/sbin/swapon/swapon.c +++ b/sbin/swapon/swapon.c @@ -377,7 +377,8 @@ swap_on_geli_args(const char *mntops) } } else if ((p = strstr(token, "notrim")) == token) { Tflag = " -T "; - } else if (strcmp(token, "sw") != 0) { + } else if (strcmp(token, "sw") != 0 && + strcmp(token, "late") != 0) { warnx("Invalid option: %s", token); free(ops); return (NULL); -- 2.11.0 From d814ad7f7c4c4b8d19fce227033922df5c50464f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 18 Apr 2015 19:38:07 +0200 Subject: [PATCH 051/257] gmountver.8: Note that GEOM mount verification has dangerous bugs Obtained from: ElectroBSD --- sbin/geom/class/mountver/gmountver.8 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sbin/geom/class/mountver/gmountver.8 b/sbin/geom/class/mountver/gmountver.8 index 4c27a652b0b0..ddb9e2b2a73f 100644 --- a/sbin/geom/class/mountver/gmountver.8 +++ b/sbin/geom/class/mountver/gmountver.8 @@ -61,6 +61,8 @@ got disconnected - it queues all the I/O requests and waits for the provider to reappear. When that happens, it attaches to it and sends the queued requests. .Pp +At least that's the theory, please note the BUGS section. +.Pp The first argument to .Nm indicates an action to be performed: @@ -119,6 +121,13 @@ If set to 0, .Nm will reattach to the device even if the device reports a different disk ID. .El +.Sh BUGS +The mount verification GEOM class can stall all the disk I/O instead +of just the device it is configured for. +The disk identification check currently has to be turned off for the class +to actually attach reappearing providers. +Obviously this is dangerous. +.El .Sh EXIT STATUS Exit status is 0 on success, and 1 if the command fails. .Sh SEE ALSO -- 2.11.0 From a3264f8851c974696efb54adb472689b2b67962c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 19 Apr 2015 22:58:49 +0200 Subject: [PATCH 052/257] Default to listening to 127.0.0.1 only Obtained from: ElectroBSD --- sbin/ggate/ggated/ggated.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/ggate/ggated/ggated.c b/sbin/ggate/ggated/ggated.c index 439725314aab..858dc0783fb1 100644 --- a/sbin/ggate/ggated/ggated.c +++ b/sbin/ggate/ggated/ggated.c @@ -1055,7 +1055,7 @@ main(int argc, char *argv[]) int ch, sfd, tmpsfd; unsigned port; - bindaddr = htonl(INADDR_ANY); + bindaddr = g_gate_str2ip("127.0.0.1"); port = G_GATE_PORT; while ((ch = getopt(argc, argv, "a:hnp:F:R:S:v")) != -1) { switch (ch) { -- 2.11.0 From 73a3742034086144c5bcb66e34ba218ff94624cb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 30 Mar 2015 15:24:05 +0200 Subject: [PATCH 053/257] ggatel: Optionally retry in case of failed reads and writes ... after waiting five seconds and reopening the device. This allows to use ggatel as a workaround against USB flakiness which can result in device disconnects that are extremely annoying if ZFS on geli is involved. To prevent data loss if multiple device disappear at the same time and reappear with different names, the disk ident is checked to confirm that the device is the expected one. As a side-effect retrying will not work for files (which have no disk ident). Obtained from: ElectroBSD --- sbin/ggate/ggatel/ggatel.c | 63 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/sbin/ggate/ggatel/ggatel.c b/sbin/ggate/ggatel/ggatel.c index 0b89df091d2a..864f3ae40d70 100644 --- a/sbin/ggate/ggatel/ggatel.c +++ b/sbin/ggate/ggatel/ggatel.c @@ -53,6 +53,7 @@ static const char *path = NULL; static int unit = G_GATE_UNIT_AUTO; static unsigned flags = 0; static int force = 0; +static unsigned retries = 0; static unsigned sectorsize = 0; static unsigned timeout = G_GATE_TIMEOUT; @@ -60,10 +61,10 @@ static void usage(void) { - fprintf(stderr, "usage: %s create [-v] [-o ] " + fprintf(stderr, "usage: %s create [-v] [-o ] [-r ] " "[-s sectorsize] [-t timeout] [-u unit] \n", getprogname()); - fprintf(stderr, " %s rescue [-v] [-o ] <-u unit> " - "\n", getprogname()); + fprintf(stderr, " %s rescue [-v] [-o ] [-r ] " + "<-u unit> \n", getprogname()); fprintf(stderr, " %s destroy [-f] <-u unit>\n", getprogname()); fprintf(stderr, " %s list [-v] [-u unit]\n", getprogname()); exit(EXIT_FAILURE); @@ -85,6 +86,7 @@ g_gatel_serve(int fd) { struct g_gate_ctl_io ggio; size_t bsize; + char ident[DISK_IDENT_SIZE]; if (g_gate_verbose == 0) { if (daemon(0, 0) == -1) { @@ -97,8 +99,15 @@ g_gatel_serve(int fd) ggio.gctl_unit = unit; bsize = sectorsize; ggio.gctl_data = malloc(bsize); + + errno = 0; + if (retries && ioctl(fd, DIOCGIDENT, ident) != 0) { + g_gate_xlog("Failed to get disk ident for %s: %s", path, + strerror(errno)); + } for (;;) { int error; + int retries_left; once_again: ggio.gctl_length = bsize; ggio.gctl_error = 0; @@ -130,6 +139,8 @@ once_again: strerror(error)); } + retries_left = retries; +retry_request: error = 0; switch (ggio.gctl_cmd) { case BIO_READ: @@ -145,6 +156,10 @@ once_again: if (pread(fd, ggio.gctl_data, ggio.gctl_length, ggio.gctl_offset) == -1) { error = errno; + g_gate_log(LOG_ERR, "Failed to read %d" + " bytes from %s: %s", + ggio.gctl_length, path, + strerror(error)); } } break; @@ -153,12 +168,44 @@ once_again: if (pwrite(fd, ggio.gctl_data, ggio.gctl_length, ggio.gctl_offset) == -1) { error = errno; + g_gate_log(LOG_ERR, "Failed to write %d bytes" + " to %s: %s", ggio.gctl_length, path, + strerror(error)); } break; default: error = EOPNOTSUPP; } + if (error && error != EOPNOTSUPP) { + if (retries_left > 0) { + char ident_new[DISK_IDENT_SIZE]; + + close(fd); + retries_left--; + sleep(5); + fd = open(path, g_gate_openflags(flags) | + O_DIRECT | O_FSYNC); + if (fd == -1) { + err(EXIT_FAILURE, "Cannot open %s", + path); + } + if (ioctl(fd, DIOCGIDENT, ident_new) != 0) { + g_gate_xlog("Failed to get disk ", + "ident for %s: %s", path, + strerror(errno)); + } + if (strcmp(ident, ident_new) != 0) { + g_gate_xlog("Disk ident for %s " + "changed from %s to %s. Reuse " + "could cause data loss.", path, + ident, ident_new); + } + g_gate_log(LOG_ERR, "Retrying after reopening " + "%s (%s)", path, ident); + goto retry_request; + } + } ggio.gctl_error = error; g_gate_ioctl(G_GATE_CMD_DONE, &ggio); } @@ -230,7 +277,7 @@ main(int argc, char *argv[]) for (;;) { int ch; - ch = getopt(argc, argv, "fo:s:t:u:v"); + ch = getopt(argc, argv, "fo:r:s:t:u:v"); if (ch == -1) break; switch (ch) { @@ -253,6 +300,14 @@ main(int argc, char *argv[]) "Invalid argument for '-o' option."); } break; + case 'r': + if (action != CREATE && action != RESCUE) + usage(); + errno = 0; + retries = strtoul(optarg, NULL, 10); + if (retries == 0 && errno != 0) + errx(EXIT_FAILURE, "Invalid retry count."); + break; case 's': if (action != CREATE) usage(); -- 2.11.0 From 81dfb369639914ed361c71d287bdb809e39cf0d5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 26 Apr 2015 17:54:14 +0200 Subject: [PATCH 054/257] ggatel.8: Document the shiny new -r option Obtained from: ElectroBSD --- sbin/ggate/ggatel/ggatel.8 | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/sbin/ggate/ggatel/ggatel.8 b/sbin/ggate/ggatel/ggatel.8 index f2eea7cf3fd5..7612b7509bd1 100644 --- a/sbin/ggate/ggatel/ggatel.8 +++ b/sbin/ggate/ggatel/ggatel.8 @@ -35,6 +35,7 @@ .Cm create .Op Fl v .Op Fl o Cm ro | wo | rw +.Op Fl r Ar retries .Op Fl s Ar sectorsize .Op Fl t Ar timeout .Op Fl u Ar unit @@ -51,6 +52,7 @@ .Cm rescue .Op Fl v .Op Fl o Cm ro | wo | rw +.Op Fl r Ar retries .Fl u Ar unit .Ar path .Sh DESCRIPTION @@ -102,6 +104,19 @@ or read-write .Pq Cm rw . Default is .Cm rw . +.It Fl r Ar retries +Number of times a failed request should be retried before forwarding +the error to the kernel. +Between retries, +.Nm ggatel +waits for five seconds and reopens the device in case it temporarily +disappeared. +The reopened device is only used if the disk identification did not +change. +This option is useful when using unreliable USB devices as geli +consumer (as long as the device loss does not cause the USB +stack to deadlock). +By default failed requests are not retried. .It Fl s Ar sectorsize Sector size for .Nm ggate @@ -144,6 +159,52 @@ umount /secret gbde detach ggate5 ggatel destroy -u 5 .Ed + +Scrub a pool on an USB device that occasionally disappears: +.Bd -literal -offset indent +$ glabel list da0 +Geom name: da0 +Providers: +1. Name: label/extreme + Mediasize: 4023385600 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + secoffset: 0 + offset: 0 + seclength: 7858175 + length: 4023385600 + index: 0 +Consumers: +1. Name: da0 + Mediasize: 4023386112 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + +$ sudo ggatel create -r 2 /dev/da0 +ggate0 +$ glabel list da0 +glabel: No such geom: da0. +glabel list ggate0 +Geom name: ggate0 +Providers: +1. Name: label/extreme + Mediasize: 4023385600 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + secoffset: 0 + offset: 0 + seclength: 7858175 + length: 4023385600 + index: 0 +Consumers: +1. Name: ggate0 + Mediasize: 4023386112 (3.7G) + Sectorsize: 512 + Mode: r0w0e0 + +$ zogftw import extreme +$ sudo zpool scrub extreme +.Ed .Sh SEE ALSO .Xr geom 4 , .Xr gbde 8 , -- 2.11.0 From 67dade27fee0c4b86a2d9d4c4c6372346daf91a8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 May 2015 18:48:20 +0200 Subject: [PATCH 055/257] g_multipath: Add sysctl to disable tasting Obtained from: ElectroBSD --- sys/geom/multipath/g_multipath.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sys/geom/multipath/g_multipath.c b/sys/geom/multipath/g_multipath.c index b461747bf356..b97d515a4b8e 100644 --- a/sys/geom/multipath/g_multipath.c +++ b/sys/geom/multipath/g_multipath.c @@ -58,6 +58,9 @@ SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, debug, CTLFLAG_RW, static u_int g_multipath_exclusive = 1; SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, exclusive, CTLFLAG_RW, &g_multipath_exclusive, 0, "Exclusively open providers"); +static u_int g_multipath_enable_tasting = 1; +SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, taste, CTLFLAG_RW, + &g_multipath_enable_tasting, 0, "Enable multipath tasting. May cause conflicts."); static enum { GKT_NIL, @@ -798,6 +801,9 @@ g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) g_topology_assert(); + if (g_multipath_enable_tasting == 0) + return (NULL); + gp = g_new_geomf(mp, "multipath:taste"); gp->start = g_multipath_start; gp->access = g_multipath_access; -- 2.11.0 From 8d6d79964cb0b83ef5d6bfe49bcad9e60624305e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 9 May 2015 14:32:31 +0200 Subject: [PATCH 056/257] motd: Customize for ElectroBSD Obtained from: ElectroBSD --- etc/motd | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/etc/motd b/etc/motd index 4dc41a744c31..3069d8ed4b20 100644 --- a/etc/motd +++ b/etc/motd @@ -1,21 +1,8 @@ -FreeBSD ?.?.? (UNKNOWN) +ElectroBSD ?.?.? (UNKNOWN) -Welcome to FreeBSD! +Welcome to ElectroBSD! -Release Notes, Errata: https://www.FreeBSD.org/releases/ -Security Advisories: https://www.FreeBSD.org/security/ -FreeBSD Handbook: https://www.FreeBSD.org/handbook/ -FreeBSD FAQ: https://www.FreeBSD.org/faq/ -Questions List: https://lists.FreeBSD.org/mailman/listinfo/freebsd-questions/ -FreeBSD Forums: https://forums.FreeBSD.org/ - -Documents installed with the system are in the /usr/local/share/doc/freebsd/ -directory, or can be installed later with: pkg install en-freebsd-doc -For other languages, replace "en" with a language code like de or fr. - -Show the version of FreeBSD installed: freebsd-version ; uname -a -Please include that output and any error messages when posting questions. -Introduction to manual pages: man man -FreeBSD directory layout: man hier +For details see: +https://www.ElectroBSD.org/ Edit /etc/motd to change this login announcement. -- 2.11.0 From b8f8f207267aaea32d5a3fd76e14073730f95beb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 14:18:43 +0200 Subject: [PATCH 057/257] release: Load usb modules through loader.conf so 'memstick' works with ELECTRO_BEER Obtained from: ElectroBSD --- release/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release/Makefile b/release/Makefile index b1dfa82ff8fd..60ffead48a54 100644 --- a/release/Makefile +++ b/release/Makefile @@ -239,6 +239,9 @@ dvd: packagesystem echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf echo vfs.mountroot.timeout=\"10\" >> ${.TARGET}/boot/loader.conf + for module in usb usb_quirk ehci umass; do \ + echo $${module}_load=\"YES\" >> ${.TARGET}/boot/loader.conf + done cp ${.CURDIR}/rc.local ${.TARGET}/etc touch ${.TARGET} -- 2.11.0 From 7b3d581299752c5e0e88d6a9f6b151c010c7c832 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 19:13:51 +0200 Subject: [PATCH 058/257] brand-fbsd.4th: Change OS name in banner to ElectroBSD While at it, suggest to resist unlawful police activities (German). Obtained from: ElectroBSD --- sys/boot/forth/brand-fbsd.4th | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sys/boot/forth/brand-fbsd.4th b/sys/boot/forth/brand-fbsd.4th index 9cd017f84a3f..0a62e98dc4d0 100644 --- a/sys/boot/forth/brand-fbsd.4th +++ b/sys/boot/forth/brand-fbsd.4th @@ -32,15 +32,17 @@ 1+ \ increase y for next time we're called ; -: brand ( x y -- ) \ "FreeBSD" [wide] logo in B/W (7 rows x 42 columns) +: brand ( x y -- ) \ "ElectroBSD" [wide] logo in B/W - s" ______ ____ _____ _____ " brand+ - s" | ____| | _ \ / ____| __ \ " brand+ - s" | |___ _ __ ___ ___ | |_) | (___ | | | |" brand+ - s" | ___| '__/ _ \/ _ \| _ < \___ \| | | |" brand+ - s" | | | | | __/ __/| |_) |____) | |__| |" brand+ - s" | | | | | | || | | |" brand+ - s" |_| |_| \___|\___||____/|_____/|_____/ " brand+ + s" ______ _ _ ____ _____ _____" brand+ + s" | ____| | | | | _ \ / ____| __ \" brand+ + s" | |__ | | ___ ___| |_ _ __ ___ | |_) | (___ | | | |" brand+ + s" | __| | |/ _ \/ __| __| '__/ _ \| _ < \___ \| | | |" brand+ + s" | |____| | __/ (__| |_| | | (_) | |_) |____) | |__| |" brand+ + s" |______|_|\___|\___|\__|_| \___/|____/|_____/|_____/" brand+ + s" Polizei-Willkuer in Deinem Land? Das erfordert Widerstand!" brand+ + s" Zu Risiken und Nebenwirkungen fragen Sie Ihren Anwalt oder" brand+ + s" die Rote Hilfe." brand+ 2drop ; -- 2.11.0 From d09dbf6ebd2f168c22c1466b27d6125cafe913bf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 19:25:56 +0200 Subject: [PATCH 059/257] beastie.4th: Think of the children and default to showing beastie instead of the sex toy Obtained from: ElectroBSD --- sys/boot/forth/beastie.4th | 4 ++-- sys/boot/forth/beastie.4th.8 | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sys/boot/forth/beastie.4th b/sys/boot/forth/beastie.4th index 752cce22a4ff..6a28605bcefd 100644 --- a/sys/boot/forth/beastie.4th +++ b/sys/boot/forth/beastie.4th @@ -64,9 +64,9 @@ variable logoY s" loader_logo" getenv dup -1 = over 0= or if dup 0= if 2drop else drop then \ getenv result unused loader_color? if - s" try-include /boot/logo-orb.4th" + s" try-include /boot/logo-beastie.4th" else - s" try-include /boot/logo-orbbw.4th" + s" try-include /boot/logo-beastiebw.4th" then else 2drop ( c-addr/u -- ) \ getenv result unused diff --git a/sys/boot/forth/beastie.4th.8 b/sys/boot/forth/beastie.4th.8 index 9f77d5db2977..3ea13c1830ea 100644 --- a/sys/boot/forth/beastie.4th.8 +++ b/sys/boot/forth/beastie.4th.8 @@ -106,10 +106,9 @@ The environment variables that effect its behavior are: Selects the desired logo in the beastie boot menu. Possible values are: .Dq Li fbsdbw , .Dq Li beastie , -.Dq Li beastiebw , +.Dq Li beastiebw (default) , .Dq Li orb , -.Dq Li orbbw -(default), and +.Dq Li orbbw , and .Dq Li none . .It Va loader_logo_x Sets the desired column position of the logo. Default is 46. -- 2.11.0 From f1091a525ceabbb69e376c7d3a5ca27089bb648c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 May 2015 19:39:56 +0200 Subject: [PATCH 060/257] menu.4th: Reduce visual noise by ditching the welcome text Obtained from: ElectroBSD --- sys/boot/forth/menu.4th | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/forth/menu.4th b/sys/boot/forth/menu.4th index e3fe0f7d776e..18d1671db6d2 100644 --- a/sys/boot/forth/menu.4th +++ b/sys/boot/forth/menu.4th @@ -470,7 +470,7 @@ also menu-infrastructure definitions \ Print the frame caption at (x,y) s" loader_menu_title" getenv dup -1 = if - drop s" Welcome to FreeBSD" + drop s" " then TRUE ( use default alignment ) s" loader_menu_title_align" getenv dup -1 <> if -- 2.11.0 From 9975bde127649f85c911e3cab49b948545f3d1a2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 17:40:22 +0200 Subject: [PATCH 061/257] logo-beastiebw: Change the fork to a toilet brush Obtained from: ElectroBSD --- sys/boot/forth/logo-beastiebw.4th | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/sys/boot/forth/logo-beastiebw.4th b/sys/boot/forth/logo-beastiebw.4th index 197099cda0bc..bce43f5b9d52 100644 --- a/sys/boot/forth/logo-beastiebw.4th +++ b/sys/boot/forth/logo-beastiebw.4th @@ -33,27 +33,27 @@ 1+ \ increase y for next time we're called ; -: logo ( x y -- ) \ B/W BSD mascot (19 rows x 34 columns) +: logo ( x y -- ) \ B/W BSD mascot with toilet brush - s" , ," logo+ - s" /( )`" logo+ - s" \ \___ / |" logo+ - s" /- _ `-/ '" logo+ - s" (/\/ \ \ /\" logo+ - s" / / | ` \" logo+ - s" O O ) / |" logo+ - s" `-^--'`< '" logo+ - s" (_.) _ ) /" logo+ - s" `.___/` /" logo+ - s" `-----' /" logo+ - s" <----. __ / __ \" logo+ - s" <----|====O)))==) \) /====|" logo+ - s" <----' `--' `.__,' \" logo+ - s" | |" logo+ - s" \ / /\" logo+ - s" ______( (_ / \______/" logo+ - s" ,' ,-----' |" logo+ - s" `--{__________)" logo+ + s" , ," logo+ + s" /( )`" logo+ + s" \ \___ / |" logo+ + s" /- _ `-/ '" logo+ + s" (/\/ \ \ /\" logo+ + s" / / | ` \" logo+ + s" O O ) / |" logo+ + s" `-^--'`< '" logo+ + s" (_.) _ ) /" logo+ + s" `.___/` /" logo+ + s" `-----' /" logo+ + s" ###### __ / __ \" logo+ + s" ######===O)))==) \) /====|" logo+ + s" ###### `--' `.__,' \" logo+ + s" | |" logo+ + s" \ / /\" logo+ + s" ______( (_ / \___/" logo+ + s" ,' ,-----' |" logo+ + s" `--{__________)" logo+ 2drop ; -- 2.11.0 From 097e5e7dee600fdeb088c4091e6b4a97f345f1b7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 18:19:19 +0200 Subject: [PATCH 062/257] logo-beastie.4th: Replace fork with toilet brush Obtained from: ElectroBSD --- sys/boot/forth/logo-beastie.4th | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/sys/boot/forth/logo-beastie.4th b/sys/boot/forth/logo-beastie.4th index 671eb5e496b2..8441c9ddafbd 100644 --- a/sys/boot/forth/logo-beastie.4th +++ b/sys/boot/forth/logo-beastie.4th @@ -35,27 +35,27 @@ 1+ \ increase y for next time we're called ; -: logo ( x y -- ) \ color BSD mascot (19 rows x 34 columns) +: logo ( x y -- ) \ color BSD mascot with toilet brush - s" @[31m, ," logo+ - s" /( )`" logo+ - s" \ \___ / |" logo+ - s" /- @[m_@[31m `-/ '" logo+ - s" (@[m/\/ \@[31m \ /\" logo+ - s" @[m/ / |@[31m ` \" logo+ - s" @[34mO O @[m) @[31m/ |" logo+ - s" @[m`-^--'@[31m`< '" logo+ - s" (_.) _ ) /" logo+ - s" `.___/` /" logo+ - s" `-----' /" logo+ - s" @[33m<----.@[31m __ / __ \" logo+ - s" @[33m<----|====@[31mO)))@[33m==@[31m) \) /@[33m====|" logo+ - s" @[33m<----'@[31m `--' `.__,' \" logo+ - s" | |" logo+ - s" \ / /\" logo+ - s" @[36m______@[31m( (_ / \______/" logo+ - s" @[36m,' ,-----' |" logo+ - s" `--{__________)@[m" logo+ + s" @[31m, ," logo+ + s" /( )`" logo+ + s" \ \___ / |" logo+ + s" /- @[m_@[31m `-/ '" logo+ + s" (@[m/\/ \@[31m \ /\" logo+ + s" @[m/ / |@[31m ` \" logo+ + s" @[34mO O @[m) @[31m/ |" logo+ + s" @[m`-^--'@[31m`< '" logo+ + s" (_.) _ ) /" logo+ + s" `.___/` /" logo+ + s" `-----' /" logo+ + s" @[37m######@[31m __ / __ \" logo+ + s" @[37m######====@[31mO)))@[37m==@[31m) \) /@[37m====|" logo+ + s" @[37m######@[31m `--' `.__,' \" logo+ + s" | |" logo+ + s" \ / /\" logo+ + s" @[36m______@[31m( (_ / \_____/" logo+ + s" @[36m,' ,-----' |" logo+ + s" `--{__________)@[m" logo+ 2drop ; -- 2.11.0 From 9a3b531f8a731ed26c652425543e742feb5e2ff9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 13:24:26 +0200 Subject: [PATCH 063/257] newsyslog.conf: Do not give world read permissions and don't compress stuff Obtained from: ElectroBSD --- etc/newsyslog.conf | 46 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/etc/newsyslog.conf b/etc/newsyslog.conf index ab595cac61db..a40949390c1e 100644 --- a/etc/newsyslog.conf +++ b/etc/newsyslog.conf @@ -10,34 +10,28 @@ # # The 'flags' field is one or more of the letters: BCDGJNUXZ or a '-'. # -# Note: some sites will want to select more restrictive protections than the -# defaults. In particular, it may be desirable to switch many of the 644 -# entries to 640 or 600. For example, some sites will consider the -# contents of maillog, messages, and lpd-errs to be confidential. In the -# future, these defaults may change to more conservative ones. -# # logfilename [owner:group] mode count size when flags [/pid_file] [sig_num] -/var/log/all.log 600 7 * @T00 J -/var/log/amd.log 644 7 100 * J -/var/log/auth.log 600 7 100 @0101T JC -/var/log/console.log 600 5 100 * J -/var/log/cron 600 3 100 * JC -/var/log/daily.log 640 7 * @T00 JN -/var/log/debug.log 600 7 100 * JC -/var/log/init.log 644 3 100 * J -/var/log/kerberos.log 600 7 100 * J -/var/log/lpd-errs 644 7 100 * JC -/var/log/maillog 640 7 * @T00 JC -/var/log/messages 644 5 100 @0101T JC -/var/log/monthly.log 640 12 * $M1D0 JN -/var/log/pflog 600 3 100 * JB /var/run/pflogd.pid -/var/log/ppp.log root:network 640 3 100 * JC -/var/log/devd.log 644 3 100 * JC -/var/log/security 600 10 100 * JC +/var/log/all.log 600 7 * @T00 +/var/log/amd.log 640 7 100 * +/var/log/auth.log 600 7 100 @0101T C +/var/log/console.log 600 5 100 * +/var/log/cron 600 3 100 * C +/var/log/daily.log 640 7 * @T00 N +/var/log/debug.log 600 7 100 * C +/var/log/init.log 640 3 100 * +/var/log/kerberos.log 600 7 100 * +/var/log/lpd-errs 640 7 100 * C +/var/log/maillog 640 7 * @T00 C +/var/log/messages 640 5 100 @0101T C +/var/log/monthly.log 640 12 * $M1D0 N +/var/log/pflog 600 3 100 * B /var/run/pflogd.pid +/var/log/ppp.log root:network 640 3 100 * C +/var/log/devd.log 640 3 100 * C +/var/log/security 600 10 100 * C /var/log/sendmail.st 640 10 * 168 BN -/var/log/utx.log 644 3 * @01T05 B -/var/log/weekly.log 640 5 * $W6D0 JN -/var/log/xferlog 600 7 100 * JC +/var/log/utx.log 640 3 * @01T05 B +/var/log/weekly.log 640 5 * $W6D0 N +/var/log/xferlog 600 7 100 * C /etc/newsyslog.conf.d/* /usr/local/etc/newsyslog.conf.d/* -- 2.11.0 From 41c46bdb7890147af8ae7838e245b3af13bc20cf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 14 May 2015 10:57:01 +0200 Subject: [PATCH 064/257] rc.d/jail: Remove obnoxious warning about 'obsolete' jail_* variables They will not be removed from ElectroBSD until a replacement exists that is usable with shell scripts like ezjail without jumping through lots of hoops. Obtained from: ElectroBSD --- etc/rc.d/jail | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/rc.d/jail b/etc/rc.d/jail index 8cf830a139a3..f9edc1a1bd79 100755 --- a/etc/rc.d/jail +++ b/etc/rc.d/jail @@ -15,7 +15,6 @@ desc="Manage system jails" rcvar="jail_enable" start_cmd="jail_start" -start_postcmd="jail_warn" stop_cmd="jail_stop" config_cmd="jail_config" console_cmd="jail_console" -- 2.11.0 From 674d77110c77a796e6b1066a927195fe93525a24 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 16:04:23 +0200 Subject: [PATCH 065/257] Don't build rcs papers until they built reproducible. Obtained from: ElectroBSD --- share/doc/psd/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/share/doc/psd/Makefile b/share/doc/psd/Makefile index 6b6d9cd727b2..9e0eed2082f5 100644 --- a/share/doc/psd/Makefile +++ b/share/doc/psd/Makefile @@ -20,7 +20,6 @@ SUBDIR= title \ 05.sysman \ 06.Clang \ 12.make \ - 13.rcs \ 15.yacc \ 16.lex \ 17.m4 \ -- 2.11.0 From 9f25b3cbb2c3fd6b9ca8eed31801dcee53db2b74 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 14 May 2015 12:34:11 +0200 Subject: [PATCH 066/257] kern.opts.mk: Disable IPFILTER, SOURCELESS_HOST and SOURCELESS_UCODE by default src.conf(5) will be updated once I find the time to figure out how to use tools/build/options/makeman. Obtained from: ElectroBSD --- sys/conf/kern.opts.mk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sys/conf/kern.opts.mk b/sys/conf/kern.opts.mk index 343b4f820161..f5913a7b5c36 100644 --- a/sys/conf/kern.opts.mk +++ b/sys/conf/kern.opts.mk @@ -33,21 +33,21 @@ __DEFAULT_YES_OPTIONS = \ FORMAT_EXTENSIONS \ INET \ INET6 \ - IPFILTER \ ISCSI \ KERNEL_SYMBOLS \ NETGRAPH \ PF \ - SOURCELESS_HOST \ - SOURCELESS_UCODE \ USB_GADGET_EXAMPLES \ ZFS __DEFAULT_NO_OPTIONS = \ EISA \ EXTRA_TCP_STACKS \ + IPFILTER \ NAND \ - OFED + OFED \ + SOURCELESS_HOST \ + SOURCELESS_UCODE # Some options are totally broken on some architectures. We disable # them. If you need to enable them on an experimental basis, you -- 2.11.0 From 6507d72c88f1c33a38d9102f19db5248e54eeddf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 15 May 2015 13:11:57 +0200 Subject: [PATCH 067/257] boot/newvers.sh branding: Change bootprog_name[] to ElectroBSD Obtained from: ElectroBSD --- sys/boot/common/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/common/newvers.sh b/sys/boot/common/newvers.sh index 167b7cc655e4..8af2304deaa8 100755 --- a/sys/boot/common/newvers.sh +++ b/sys/boot/common/newvers.sh @@ -40,6 +40,6 @@ u=${USER-root} h=${HOSTNAME-`hostname`} t=`date` #r=`head -n 6 $1 | tail -n 1 | awk -F: ' { print $1 } '` r=`awk -F: ' /^[0-9]\.[0-9]+:/ { print $1; exit }' $1` -echo "char bootprog_info[] = \"FreeBSD/${3} ${2}, Revision ${r}\\n(${t} ${u}@${h})\\n\";" > $tempfile +echo "char bootprog_info[] = \"ElectroBSD/${3} ${2}, Revision ${r}\\n(${t} ${u}@${h})\\n\";" > $tempfile echo "unsigned bootprog_rev = ${r%%.*}${r##*.};" >> $tempfile mv $tempfile vers.c -- 2.11.0 From 5f835c891de8d6b18d155df14ed3ba892e0bc1a0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 15:59:58 +0200 Subject: [PATCH 068/257] release: Default to not distributing the ports tree Allow to overwrite this with WITH_PORTS Obtained from: ElectroBSD --- release/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/Makefile b/release/Makefile index 60ffead48a54..a1c89cee63e6 100644 --- a/release/Makefile +++ b/release/Makefile @@ -23,9 +23,9 @@ # DOCDIR: location of doc tree (default: /usr/doc) # XTRADIR: xtra-bits-dir argument for /mkisoimages.sh # NOPKG: if set, do not distribute third-party packages -# NOPORTS: if set, do not distribute ports tree # NOSRC: if set, do not distribute source tree # NODOC: if set, do not generate release documentation +# WITH_PORTS: if set, distribute ports tree provided it exists # WITH_DVD: if set, generate dvd1.iso # WITH_COMPRESSED_IMAGES: if set, compress installation images with xz(1) # (uncompressed images are not removed) @@ -77,7 +77,7 @@ VOLUME_LABEL= FreeBSD_Install .if !exists(${DOCDIR}) NODOC= true .endif -.if !exists(${PORTSDIR}) +.if !exists(${PORTSDIR}) || !defined(WITH_PORTS) NOPORTS= true .endif -- 2.11.0 From cc3cfcfb1d2b2c4701aea27d4291f1b1d809124e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 15:26:44 +0200 Subject: [PATCH 069/257] sys/boot/common/newvers.sh: Allow to overwrite the date to make boot loader binaries reproducible Obtained from: ElectroBSD --- sys/boot/common/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/common/newvers.sh b/sys/boot/common/newvers.sh index 8af2304deaa8..3f4be41f5c38 100755 --- a/sys/boot/common/newvers.sh +++ b/sys/boot/common/newvers.sh @@ -36,7 +36,7 @@ tempfile=$(mktemp tmp.XXXXXX) || exit trap "rm -f $tempfile" EXIT INT TERM LC_ALL=C; export LC_ALL -u=${USER-root} h=${HOSTNAME-`hostname`} t=`date` +u=${USER-root} h=${HOSTNAME-`hostname`} t=${DATE-`date`} #r=`head -n 6 $1 | tail -n 1 | awk -F: ' { print $1 } '` r=`awk -F: ' /^[0-9]\.[0-9]+:/ { print $1; exit }' $1` -- 2.11.0 From 150bcb572645ce7aec9e683a0bb09bb9181ec5b9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 15:36:48 +0200 Subject: [PATCH 070/257] sys/conf/newvers.sh: Allow to overwrite the build date embedded into the kernel This is a required step to get reproducible builds. Obtained from: ElectroBSD --- sys/conf/newvers.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 82b136426e6f..82772244ce38 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -113,7 +113,12 @@ v=`cat version` u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} -if [ -n "$SOURCE_DATE_EPOCH" ]; then +if [ -n "${DATE}" ]; then + # SOURCE_DATE_EPOCH was added upstream in r291691 + # but the ElectroBSD build goo is still setting DATE + # which has a different format. + t=${DATE} +elif [ -n "$SOURCE_DATE_EPOCH" ]; then if ! t=`date -r $SOURCE_DATE_EPOCH 2>/dev/null`; then echo "Invalid SOURCE_DATE_EPOCH" >&2 exit 1 -- 2.11.0 From 31f52a3d176caddb7dc1a8c045fb0c069eded255 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 19:10:44 +0200 Subject: [PATCH 071/257] amd64/make-memstick.sh: Create more 'predictable' images According to the mkimg man page, the "-y option is used for testing purposes only and is not to be used in production", but it feels good to be a gangsta. Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 52e17cc97b5f..d9590d1d08cb 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -38,6 +38,6 @@ fi rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local -mkimg -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} rm ${2}.part -- 2.11.0 From 6900d7c281bf8282ed9c9c010612579a5275b992 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 19:17:14 +0200 Subject: [PATCH 072/257] release/Makefile: Don't create matroshka src tarballs that contain other tarballs ... if DESTDIR isn't set to a reasonable value. While at it, exclude *.orig files as well. Obtained from: ElectroBSD --- release/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index a1c89cee63e6..fbc252ea486f 100644 --- a/release/Makefile +++ b/release/Makefile @@ -148,7 +148,9 @@ src.txz: mkdir -p ${DISTDIR}/usr ln -fs ${WORLDDIR} ${DISTDIR}/usr/src cd ${DISTDIR} && tar cLvf - --exclude .svn --exclude .zfs \ - --exclude .git --exclude @ --exclude usr/src/release/dist usr/src | \ + --exclude .git --exclude @ --exclude usr/src/release/dist \ + --exclude "usr/src/release/*.txz" \ + --exclude "usr/src/release/*.orig" usr/src | \ ${XZ_CMD} > ${.OBJDIR}/src.txz ports.txz: -- 2.11.0 From b4d871d28cda1e1b15653d2aafceb57c9480dc82 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 18:45:24 +0200 Subject: [PATCH 073/257] release: Use a hack to recreate dist tarballs with reproducible timestamps Obtained from: ElectroBSD --- release/Makefile | 3 ++ release/scripts/tar-time-reset.sh | 64 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100755 release/scripts/tar-time-reset.sh diff --git a/release/Makefile b/release/Makefile index fbc252ea486f..627b6ea6bd7d 100644 --- a/release/Makefile +++ b/release/Makefile @@ -266,6 +266,9 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} + for tarball in *.txz; do \ + sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ + done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} diff --git a/release/scripts/tar-time-reset.sh b/release/scripts/tar-time-reset.sh new file mode 100755 index 000000000000..8186fd8d3e26 --- /dev/null +++ b/release/scripts/tar-time-reset.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# This script resets the timestamps in a given tarfile to hopefully make +# it reproducible. This is a rather wasteful approach, but works for now. +# +# A better solution would be to patch bsdtar to optionally use a fixed +# time (without having to clown around with mtree specs). +# +########################################################################## + +main() { + local tarfile="${1}" \ + tempdir mtree_spec + + if [ $# -ne 1 ]; then + echo "$0 /path/to/tarfile" + exit 1 + fi + + if [ -z "${tarfile}" ]; then + echo "No tar file given" + return 1 + fi + tarfile="$(realpath "$tarfile")" + + tempdir=$(mktemp -d) || return 1 + mtree_spec=$(mktemp) || return 1 + + echo "Extracting tarfile ${tarfile}" + (cd "${tempdir}" && tar xvf "${tarfile}") || return 1 + + echo "Ditching original tarfile ${tarfile}" + rm "${tarfile}" + + echo "Creating mtree spec in ${mtree_spec}" + (cd "${tempdir}" && mtree -L -c -k time) | \ + sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${mtree_spec}" + + echo "Creating tarfile ${tarfile}" + (cd "${tempdir}" && tar acLvf "${tarfile}" @"${mtree_spec}") || return 1 + + echo "Ditching ${tempdir}" + rm -r "${tempdir}" || return 1 + echo "Ditching ${mtree_spec}" + rm "${mtree_spec}" || return 1 +} + +main "${@}" -- 2.11.0 From 235a24122f4071033a18ce360b6f19a97454c88a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 19 May 2015 16:06:01 +0200 Subject: [PATCH 074/257] release/amd64/make-memstick.sh: Use reproducible timestamps for the makefs image Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index d9590d1d08cb..a5d88e166e23 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -30,13 +30,15 @@ fi echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local -makefs -B little -o label=FreeBSD_Install ${2}.part ${1} +mtree -c -k time -p "${1}" | sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${2}.mtree" +makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" exit 1 fi rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local +rm "${2}.mtree" mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} rm ${2}.part -- 2.11.0 From b95665989aaa4e993e0c4f197c4061ac75d2b6a6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 14:25:17 +0200 Subject: [PATCH 075/257] Remove build timestamps from ntp* binaries again This was already done by r195626 a couple of years ago apparently the change got lost while importing an update from the vendor. Obtained from: ElectroBSD --- contrib/ntp/scripts/build/mkver.in | 2 -- 1 file changed, 2 deletions(-) mode change 100644 => 100755 contrib/ntp/scripts/build/mkver.in diff --git a/contrib/ntp/scripts/build/mkver.in b/contrib/ntp/scripts/build/mkver.in old mode 100644 new mode 100755 index 3aef1c816a7f..badd2cda8c1f --- a/contrib/ntp/scripts/build/mkver.in +++ b/contrib/ntp/scripts/build/mkver.in @@ -15,8 +15,6 @@ case "@VER_SUFFIX@" in *) ConfStr="${ConfStr}-@VER_SUFFIX@" ;; esac -ConfStr="$ConfStr `LC_TIME=C TZ=UTC date`" - if [ ! -f .version ]; then echo 0 > .version fi -- 2.11.0 From 491bb4765eb18812be235aaa1f5537ff214f0feb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 20 May 2015 12:43:25 +0200 Subject: [PATCH 076/257] usr.sbin/ntp: Allow to set MKREPRO_DATE and MKREPRO_TIME to get reproducible builds Obtained from: ElectroBSD --- usr.sbin/ntp/libntp/Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/usr.sbin/ntp/libntp/Makefile b/usr.sbin/ntp/libntp/Makefile index 74babee5d841..f7c82f345af4 100644 --- a/usr.sbin/ntp/libntp/Makefile +++ b/usr.sbin/ntp/libntp/Makefile @@ -85,6 +85,13 @@ CFLAGS+= -I${.CURDIR}/../../../contrib/ntp/include \ CFLAGS+= -DHAVE_BSD_NICE -DHAVE_STDINT_H +.if defined(MKREPRO_DATE) +CFLAGS+= -DMKREPRO_DATE="\"${MKREPRO_DATE}\"" +.endif +.if defined(MKREPRO_TIME) +CFLAGS+= -DMKREPRO_TIME="\"${MKREPRO_TIME}\"" +.endif + CLEANFILES+= .version version.c version.c: -- 2.11.0 From 8682527b84a578abadb7b7427901fffd44f70acf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 22 May 2015 11:33:50 +0200 Subject: [PATCH 077/257] release: Allow to build the mtree spec for mergemaster reproducible This relies on NetBSD mtree which has been the default for a while now. Obtained from: ElectroBSD --- release/scripts/mm-mtree.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/release/scripts/mm-mtree.sh b/release/scripts/mm-mtree.sh index 620ab6dd962c..279220b78319 100755 --- a/release/scripts/mm-mtree.sh +++ b/release/scripts/mm-mtree.sh @@ -146,8 +146,12 @@ find ${TEMPROOT} -type f -size 0 -delete 2>/dev/null find -d ${TEMPROOT} -type d -empty -delete 2>/dev/null # Build the mtree database in a temporary location. +# The second mtree call is used to get a reproducible result +# without embedded hostname, user name and creation timestamp. +# XXX: Might be obsolete after r301584. MTREENEW=`mktemp -t mergemaster.mtree` -mtree -nci -p ${TEMPROOT} -k size,md5digest > ${MTREENEW} 2>/dev/null +mtree -mci -p ${TEMPROOT} -k size,md5digest 2>/dev/null | \ + mtree -C -k all > ${MTREENEW} if [ -s "${MTREENEW}" ]; then echo "*** Saving mtree database for future upgrades" -- 2.11.0 From a961f85f472124db3e1b960ac8b3b1f7bd85a7f1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 14 May 2015 12:07:25 +0200 Subject: [PATCH 078/257] Make reproducing builds more convenient ... by setting the various variables based on the environment variable REPRO_SEED. Obtained from: ElectroBSD --- Makefile.inc1 | 14 +++++++--- release/Makefile | 2 ++ share/mk/src.reproducible-build.mk | 54 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 share/mk/src.reproducible-build.mk diff --git a/Makefile.inc1 b/Makefile.inc1 index 967325f8161c..fe506aed21dd 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -48,6 +48,8 @@ .error "Both TARGET and TARGET_ARCH must be defined." .endif +.include "share/mk/src.reproducible-build.mk" + SRCDIR?= ${.CURDIR} LOCALBASE?= /usr/local @@ -787,17 +789,23 @@ WMAKE_TGTS+= build${libcompat} buildworld: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue .PHONY .ORDER: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue -buildworld_prologue: .PHONY +buildworld_prologue: .PHONY reproducible_build_hint @echo "--------------------------------------------------------------" @echo ">>> World build started on `LC_ALL=C date`" @echo "--------------------------------------------------------------" -buildworld_epilogue: .PHONY +buildworld_epilogue: .PHONY reproducible_build_hint @echo @echo "--------------------------------------------------------------" @echo ">>> World build completed on `LC_ALL=C date`" @echo "--------------------------------------------------------------" +reproducible_build_hint: + @echo "--------------------------------------------------------------" + @echo ">>> To reproduce this build:" + @echo ">>> export REPRO_SEED=$${REPRO_SEED}" + @echo "--------------------------------------------------------------" + # # We need to have this as a target because the indirection between Makefile # and Makefile.inc1 causes the correct PATH to be used, rather than a @@ -1167,7 +1175,7 @@ ${WMAKE_TGTS:N_worldtmp:Nbuild${libcompat}} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE # # Builds all kernels defined by BUILDKERNELS. # -buildkernel: .MAKE .PHONY +buildkernel: .MAKE .PHONY reproducible_build_hint .if empty(BUILDKERNELS:Ndummy) @echo "ERROR: Missing kernel configuration file(s) (${KERNCONF})."; \ false diff --git a/release/Makefile b/release/Makefile index 627b6ea6bd7d..7fa748246b37 100644 --- a/release/Makefile +++ b/release/Makefile @@ -36,6 +36,8 @@ # TARGET/TARGET_ARCH: architecture of built release # +.include "../share/mk/src.reproducible-build.mk" + WORLDDIR?= ${.CURDIR}/.. PORTSDIR?= /usr/ports DOCDIR?= /usr/doc diff --git a/share/mk/src.reproducible-build.mk b/share/mk/src.reproducible-build.mk new file mode 100644 index 000000000000..dc4f4df93507 --- /dev/null +++ b/share/mk/src.reproducible-build.mk @@ -0,0 +1,54 @@ +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# Make the build reproducible by exporting a bunch of variables, +# potentionally using an already-set REPRO_SEED as input. +# +# The variable names are somewhat stupid, mostly because we +# are using existing ones. +# +########################################################################## + +TZ= "UTC" +.export TZ + +.if ! defined(REPRO_SEED) +# XXX: Currently we don't add the kernel version number to the repro +# seed because we expect a clean object tree in which case it +# will reproducible be 0. If the object tree of a the +# build-to-reproduce was actually unclean, KERNEL_VERSION_NUMBER +# has to be set to a matching value. +REPRO_SEED!= echo $$(id -un):$$(hostname):$$(date +%s) +.export REPRO_SEED +.else +.if ! defined(KERNEL_VERSION_NUMBER) +KERNEL_VERSION_NUMBER=0 +.export KERNEL_VERSION_NUMBER +.endif +.endif + +USER!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 1 +HOSTNAME!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 2 +EPOCH_DATE!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 3 + +DATE!= date -r ${EPOCH_DATE} +# These two probably are no longer necessary after r285701 +MKREPRO_DATE!= date -r ${EPOCH_DATE} +"%b %d %Y" +MKREPRO_TIME!= date -r ${EPOCH_DATE} +%H:%M:%S + +.for v in REPRO_SEED USER HOSTNAME EPOCH_DATE DATE MKREPRO_DATE MKREPRO_TIME +.export $v +.endfor -- 2.11.0 From 57f1f29b500d5f6e8531b332f7619a1972a2976f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 23 May 2015 20:13:18 +0200 Subject: [PATCH 079/257] contrib/groff/mdate.sh: Use ${EPOCH_DATE} when set While this is silly, using the time the man pages have been last checked out from the VCS is silly as well, so this commit doesn't increase the total amount of sillyness in the system. Also we are talking about GNU roff here, so hopefully this stuff will be garbage-collected in the near future. Obtained from: ElectroBSD --- contrib/groff/mdate.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/groff/mdate.sh b/contrib/groff/mdate.sh index 4a26e6e80bad..615abc784123 100755 --- a/contrib/groff/mdate.sh +++ b/contrib/groff/mdate.sh @@ -1,12 +1,17 @@ #! /bin/sh # Print the modification date of $1 `nicely'. +# If ${EPOCH_DATE} is set, it will be used instead. # Don't want foreign dates. LANGUAGE= LC_ALL=C; export LC_ALL +if [ -n "${EPOCH_DATE}" ]; then + date -r "${EPOCH_DATE}" +"%d %B %Y" + exit 0 +fi (date; if ls -L /dev/null 1>/dev/null 2>&1; then ls -L -l $1; else ls -l $1; fi -- 2.11.0 From 2876d885a119a2647f2f0bddecbf8f78bdede0a8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 May 2015 18:01:31 +0200 Subject: [PATCH 080/257] Fake modification time smarter for everything but EXTRA_PACKAGES (XXX) While at it, remove duplicated slashes in the METALOG as they result in missing files. XXX 2017-01: It's not clear if this is still the case, but embedding duplicated slashes in the dist files is also aesthetically unpleasing. Mark two suspicious mtree spec modifications as such. XXX: investigate. Obtained from: ElectroBSD --- Makefile.inc1 | 20 ++++++++++++++++++++ release/Makefile | 6 +++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index fe506aed21dd..c27fefbe8ad1 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1019,6 +1019,14 @@ distributeworld installworld stageworld: _installcheck_world .PHONY find ${DESTDIR}/${DISTDIR}/${dist} -mindepth 1 -type d -empty -delete .endfor .if defined(NO_ROOT) + @# Post process METALOG: add fake timestamps and, if necessary, + @# remove duplicated slashes which can occur if DISTDIR is undefined. + @# Keeping them results in missing files in the distribution tarballs. +.if defined(EPOCH_DATE) + sed -E -e 's@time=[0-9\.]+@@' \ + -e 's@(type=)@uid=0 gid=0 time=${EPOCH_DATE}.0 \1@' \ + -e 's@//@/@g' -i '.bak' ${METALOG} +.endif .for dist in base ${EXTRA_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that @@ -1300,6 +1308,12 @@ distributekernel distributekernel.debug: .PHONY packagekernel: .PHONY .if defined(NO_ROOT) .if !defined(NO_INSTALLKERNEL) +.if defined(EPOCH_DATE) +# XXX: Is this really necessary given that we already modify the METALOG itself? + sed -E -e 's@time=[0-9\.]+@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + -e 's@//@/@g' \ + -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.meta +.endif cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --exclude '*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.meta | \ @@ -1311,6 +1325,12 @@ packagekernel: .PHONY ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel-dbg.txz .if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} +# XXX: See XXX above +.if defined(EPOCH_DATE) + sed -E -e 's@time=[0-9\.]+@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + -e 's@//@/@g' \ + -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta +.endif cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --exclude '*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \ diff --git a/release/Makefile b/release/Makefile index 7fa748246b37..e227d19aa559 100644 --- a/release/Makefile +++ b/release/Makefile @@ -56,6 +56,10 @@ TARGET_ARCH= ${TARGET} IMAKE= ${MAKE} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET} DISTDIR= dist +# Enable mtree spec usage so we can fake the modification time. +NO_ROOT=1 +.export NO_ROOT + # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) .for _V in TYPE BRANCH REVISION @@ -268,7 +272,7 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} - for tarball in *.txz; do \ + for tarball in ${EXTRA_PACKAGES}; do \ sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST -- 2.11.0 From 22159395acebf5b5715b2c29eba295142b4344e5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 1 Jun 2015 13:56:05 +0200 Subject: [PATCH 081/257] release/Makefile: Build the src.txz only once ... and also enforce reproducible source ownership and permissions. If taring the sources fails, keep the mtree spec. Keep release/scripts/tar-time-reset.sh for now but update a comment to make it obvious that the script isn't used anymore. Obtained from: ElectroBSD --- release/Makefile | 31 +++++++++++++++++++++++-------- release/scripts/tar-time-reset.sh | 8 +++----- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/release/Makefile b/release/Makefile index e227d19aa559..acebba7924c3 100644 --- a/release/Makefile +++ b/release/Makefile @@ -153,11 +153,29 @@ kernel.txz: src.txz: mkdir -p ${DISTDIR}/usr ln -fs ${WORLDDIR} ${DISTDIR}/usr/src - cd ${DISTDIR} && tar cLvf - --exclude .svn --exclude .zfs \ - --exclude .git --exclude @ --exclude usr/src/release/dist \ - --exclude "usr/src/release/*.txz" \ - --exclude "usr/src/release/*.orig" usr/src | \ - ${XZ_CMD} > ${.OBJDIR}/src.txz +# Create an mtree spec with faked timestamps so we get a reproducible +# tar file. We do not use tar for this because its mtree generator +# appears to be buggy and exits with an memory allocation failure. +# +# It's important that the excluded paths start with "./", otherwise +# file locations are not recorded correctly, and, for example, +# usr/src/usr.sbin appears as usr/src/release/usr.sbin in the +# tar file. Only the shadow knows if that's a bug or a feature. + echo "./usr/src/.git" >${.OBJDIR}/mtree-exclude + echo "./usr/src/release/dist" >>${.OBJDIR}/mtree-exclude + echo "./usr/src/release/src.mtree" >>${.OBJDIR}/mtree-exclude + echo "./usr/src/release/mtree-exclude" >>${.OBJDIR}/mtree-exclude + cd ${DISTDIR} && mtree -c -L -k time -X ${.OBJDIR}/mtree-exclude | \ + mtree -C | \ + sed -E -e 's@time=[0-9]+\.[0-9]+@time=${EPOCH_DATE}.0 uid=0 gid=0@' \ + -e 's@(type=dir)@\1 mode=0550@' \ + -e 's@(type=file)@\1 mode=0440@' \ + -e 's@//@/@g' \ + > ${.OBJDIR}/src.mtree + rm ${.OBJDIR}/mtree-exclude + cd ${DISTDIR} && tar cLvf - @${.OBJDIR}/src.mtree \ + | ${XZ_CMD} > ${.OBJDIR}/src.txz && \ + rm ${.OBJDIR}/src.mtree ports.txz: mkdir -p ${DISTDIR}/usr @@ -272,9 +290,6 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} - for tarball in ${EXTRA_PACKAGES}; do \ - sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ - done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} diff --git a/release/scripts/tar-time-reset.sh b/release/scripts/tar-time-reset.sh index 8186fd8d3e26..f1b84340286d 100755 --- a/release/scripts/tar-time-reset.sh +++ b/release/scripts/tar-time-reset.sh @@ -17,11 +17,9 @@ ########################################################################## # # This script resets the timestamps in a given tarfile to hopefully make -# it reproducible. This is a rather wasteful approach, but works for now. -# -# A better solution would be to patch bsdtar to optionally use a fixed -# time (without having to clown around with mtree specs). -# +# it reproducible. As this is a rather wasteful approach the script is +# no longer used. It hasn't been removed yet as it may be useful for +# testing purposes. ########################################################################## main() { -- 2.11.0 From 86d55cbc1b31b12c804bde639ef9c5394d315152 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 May 2015 10:27:48 +0200 Subject: [PATCH 082/257] Add image-checksum.sh ... which calculates a checksum of the reproducible parts of an memstick image. Obtained from: ElectroBSD --- release/scripts/image-checksum.sh | 156 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100755 release/scripts/image-checksum.sh diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh new file mode 100755 index 000000000000..6f79a81ae37a --- /dev/null +++ b/release/scripts/image-checksum.sh @@ -0,0 +1,156 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# image-checksum.sh /path/to/memstick.img +# +# Unfortunately the memstick target currently does not create reproducible +# ElectroBSD images due to unreproducible differences in the file system +# layer of the data partition. +# +# To be able to (sort of) compare memstick images anyway, this script +# produces a "partial image checksum" that is based on the partition layout, +# the checksum of the boot code partition and an mtree spec of the data +# partition which includes checksums, sizes and timestamps for all the +# files. +# +# A memstick image whose "partial checksum" matches the one of another +# image can be totally considered to be nearly as trustworthy. Obviously +# that's a somewhat worthless property, it is thus recommended that you +# rebuild the potentionally malicious image using a trusted operating +# system first. After you've done this, potentionally malicious differences +# in the unchecked parts should be gone. +# +# Just kidding, image-checksum.sh is only intended to regression-test +# the ElectroBSD build system. +# +# Also note that this script relies on non-standardized output of other +# tools which might occasionally change. To be able to reproduce partial +# image checksums you thus need a userland that is close enough to the +# one that was used to create the original version. +# +########################################################################## + +UFS_PARTITION=p2 +EXPECTED_PARTITIONS=2 +MOUNTPOINT=/mnt +VERBOSE=0 +MTREE_KEYWORDS=size,time,uid,gid,sha256 + +verbose_log() { + local message="$*" + if [ "${VERBOSE}" = 0 ]; then + return + fi + echo "${message}" +} + +create_mtree_spec_file() { + local md_unit spec_file + + md_unit="${1}" + spec_file="${2}" + + verbose_log "Mounting /dev/md${md_unit}${UFS_PARTITION} at ${MOUNTPOINT}" + mount -o ro "/dev/md${md_unit}${UFS_PARTITION}" "${MOUNTPOINT}" || return 1 + + verbose_log "Running mtree, saving spec in ${spec_file}" + mtree -c -k "${MTREE_KEYWORDS}" -p "${MOUNTPOINT}" | mtree -C -k all > "${spec_file}" || return 1 + + verbose_log "Unmounting ${MOUNTPOINT} ..." + umount "${MOUNTPOINT}" || return 1 +} + +partition_count_acceptable() { + local md_unit="${1}" + + # Verify that there are exactly two partitions present + partitions=$(gpart show -r -p "md${md_unit}" | grep -c "md${md_unit}"p) + if [ "${partitions}" != "${EXPECTED_PARTITIONS}" ]; then + echo "Invalid number of partitions: ${partitions}" + return 1; + fi +} + +main() { + local image_file \ + args md_unit spec_file gpart_file + + args=$(getopt v $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -v) + VERBOSE=1 + shift + ;; + --) + shift; break + ;; + esac + done + + image_file=${1} + if [ -z "${image_file}" ]; then + echo "No image file provided" + return 1 + fi + spec_file="${image_file}.mtree" + if [ -f "${spec_file}" ]; then + echo "Spec file ${spec_file} already exists" + return 1 + fi + gpart_file="${image_file}.gpart" + if [ -f "${spec_file}" ]; then + echo "gpart file ${gpart_file} already exists" + return 1 + fi + + md_unit=$(mdconfig -o readonly -n -f "${image_file}") + if [ $? != 0 ]; then + return 1 + fi + + partition_count_acceptable "${md_unit}" || return 1 + + if [ ! -f "${spec_file}" ]; then + create_mtree_spec_file "${md_unit}" "${spec_file}" || return 1 + fi + if [ ! -f "${gpart_file}" ]; then + gpart list "md${md_unit}" | sed -E -e "s@(: md)${md_unit}@\1X@" > "${gpart_file}" + fi + + gpart_checksum=$(sha256 -q "${gpart_file}") + verbose_log "gpart checksum: ${gpart_checksum}" + + mdconfig -d -u "${md_unit}" || return 1 + + bootcode_checksum=$(dd if=/dev/md${md_unit}p1 2>/dev/null | sha256) + verbose_log "Boot code checksum: ${bootcode_checksum}" + mtree_checksum=$(sha256 -q "${spec_file}") + verbose_log "mtree checksum: ${mtree_checksum}" + + weak_image_checksum=$(echo "${gpart_checksum} ${bootcode_checksum} ${mtree_checksum}" | sha256) + echo "Partial image checksum: ${weak_image_checksum}" + +} + +main "${@}" -- 2.11.0 From bf87fe65959851dcaa4f924c029a138486991b6c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 19:54:10 +0200 Subject: [PATCH 083/257] image-checksum: Add -r flag to reuse cache files Obtained from: ElectroBSD --- release/scripts/image-checksum.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index 6f79a81ae37a..1940921b44a1 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -50,6 +50,7 @@ EXPECTED_PARTITIONS=2 MOUNTPOINT=/mnt VERBOSE=0 MTREE_KEYWORDS=size,time,uid,gid,sha256 +REUSE_EXISTING_CACHE_FILES=false verbose_log() { local message="$*" @@ -90,7 +91,7 @@ main() { local image_file \ args md_unit spec_file gpart_file - args=$(getopt v $*) + args=$(getopt rv $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -98,6 +99,10 @@ main() { set -- ${args} while true; do case "$1" in + -r) + REUSE_EXISTING_CACHE_FILES=true + shift + ;; -v) VERBOSE=1 shift @@ -116,12 +121,12 @@ main() { spec_file="${image_file}.mtree" if [ -f "${spec_file}" ]; then echo "Spec file ${spec_file} already exists" - return 1 + ${REUSE_EXISTING_CACHE_FILES} || return 1 fi gpart_file="${image_file}.gpart" if [ -f "${spec_file}" ]; then echo "gpart file ${gpart_file} already exists" - return 1 + ${REUSE_EXISTING_CACHE_FILES} || return 1 fi md_unit=$(mdconfig -o readonly -n -f "${image_file}") -- 2.11.0 From 3516079a0f4d5b0d97614094084e1c9a167218ae Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 20:20:01 +0200 Subject: [PATCH 084/257] image-checksum.sh: Allow to overwrite the mtree flags Obtained from: ElectroBSD --- release/scripts/image-checksum.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index 1940921b44a1..cc31e9055fe3 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -91,7 +91,7 @@ main() { local image_file \ args md_unit spec_file gpart_file - args=$(getopt rv $*) + args=$(getopt m:rv $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -99,6 +99,11 @@ main() { set -- ${args} while true; do case "$1" in + -m) + shift + MTREE_KEYWORDS="${1}" + shift + ;; -r) REUSE_EXISTING_CACHE_FILES=true shift -- 2.11.0 From aa5bc0b656967bfd706bd5d2648e7ce29415b2c9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 31 Aug 2015 10:51:16 +0200 Subject: [PATCH 085/257] release/scripts/image-checksum.sh: Allow to checksum multiple images at once Obtained from: ElectroBSD --- release/scripts/image-checksum.sh | 79 ++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index cc31e9055fe3..30ac62a6c168 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -87,42 +87,11 @@ partition_count_acceptable() { fi } -main() { +generate_partial_image_checksum() { local image_file \ - args md_unit spec_file gpart_file - - args=$(getopt m:rv $*) - if [ $? -ne 0 ]; then - echo 'You are doing it wrong: Invalid flag specified' - exit 2 - fi - set -- ${args} - while true; do - case "$1" in - -m) - shift - MTREE_KEYWORDS="${1}" - shift - ;; - -r) - REUSE_EXISTING_CACHE_FILES=true - shift - ;; - -v) - VERBOSE=1 - shift - ;; - --) - shift; break - ;; - esac - done + md_unit spec_file gpart_file - image_file=${1} - if [ -z "${image_file}" ]; then - echo "No image file provided" - return 1 - fi + image_file="${1}" spec_file="${image_file}.mtree" if [ -f "${spec_file}" ]; then echo "Spec file ${spec_file} already exists" @@ -159,8 +128,48 @@ main() { verbose_log "mtree checksum: ${mtree_checksum}" weak_image_checksum=$(echo "${gpart_checksum} ${bootcode_checksum} ${mtree_checksum}" | sha256) - echo "Partial image checksum: ${weak_image_checksum}" + echo "Partial image checksum for ${image_file}: ${weak_image_checksum}" +} +main() { + local image_file \ + args + + args=$(getopt m:rv $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -m) + shift + MTREE_KEYWORDS="${1}" + shift + ;; + -r) + REUSE_EXISTING_CACHE_FILES=true + shift + ;; + -v) + VERBOSE=1 + shift + ;; + --) + shift; break + ;; + esac + done + + if [ -z "${1}" ]; then + echo "No image file provided" + return 1 + fi + + for image_file in "${@}"; do + generate_partial_image_checksum "${image_file}" || return 1 + done } main "${@}" -- 2.11.0 From f558bb6bb756df93d3b0bdca163f27e115a05d14 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 30 May 2015 14:38:48 +0200 Subject: [PATCH 086/257] sys: Do not embed the compiler version in the kernel binary ... as it makes reproducing the binary with a different compiler more complicated. In case of ElectroBSD the compiler used can be usually deduced from the uname output anyway as the upstream revision is part of the fake hostname when using reproduce.sh. Obtained from: ElectroBSD --- sys/conf/newvers.sh | 2 -- sys/kern/init_main.c | 1 - sys/kern/kern_mib.c | 3 --- sys/sys/systm.h | 1 - 4 files changed, 7 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 82772244ce38..7c9c0474d218 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -127,7 +127,6 @@ else t=`date` fi i=`${MAKE:-make} -V KERN_IDENT` -compiler_v=$($(${MAKE:-make} -V CC) -v 2>&1 | grep -w 'version') for dir in /usr/bin /usr/local/bin; do if [ ! -z "${svnversion}" ] ; then @@ -286,7 +285,6 @@ $COPYRIGHT char sccs[sizeof(SCCSSTR) > 128 ? sizeof(SCCSSTR) : 128] = SCCSSTR; char version[sizeof(VERSTR) > 256 ? sizeof(VERSTR) : 256] = VERSTR; -char compiler_version[] = "${compiler_v}"; char ostype[] = "${TYPE}"; char osrelease[sizeof(RELSTR) > 32 ? sizeof(RELSTR) : 32] = RELSTR; int osreldate = ${RELDATE}; diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 85382cdd8c51..c416c9acf7c6 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -341,7 +341,6 @@ print_version(void *data __unused) while (len > 0 && version[len - 1] == '\n') len--; printf("%.*s %s\n", len, version, machine); - printf("%s\n", compiler_version); } SYSINIT(announce, SI_SUB_COPYRIGHT, SI_ORDER_FIRST, print_caddr_t, diff --git a/sys/kern/kern_mib.c b/sys/kern/kern_mib.c index 875d2b97c998..ab013666306b 100644 --- a/sys/kern/kern_mib.c +++ b/sys/kern/kern_mib.c @@ -97,9 +97,6 @@ SYSCTL_INT(_kern, KERN_OSREV, osrevision, CTLFLAG_RD|CTLFLAG_CAPRD, SYSCTL_STRING(_kern, KERN_VERSION, version, CTLFLAG_RD|CTLFLAG_MPSAFE, version, 0, "Kernel version"); -SYSCTL_STRING(_kern, OID_AUTO, compiler_version, CTLFLAG_RD|CTLFLAG_MPSAFE, - compiler_version, 0, "Version of compiler used to compile kernel"); - SYSCTL_STRING(_kern, KERN_OSTYPE, ostype, CTLFLAG_RD|CTLFLAG_MPSAFE| CTLFLAG_CAPRD, ostype, 0, "Operating system type"); diff --git a/sys/sys/systm.h b/sys/sys/systm.h index caffd7baafdf..90d633684497 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -50,7 +50,6 @@ extern int suspend_blocked; /* block suspend due to pending shutdown */ extern int rebooting; /* kern_reboot() has been called. */ extern const char *panicstr; /* panic message */ extern char version[]; /* system version */ -extern char compiler_version[]; /* compiler version */ extern char copyright[]; /* system copyright */ extern int kstack_pages; /* number of kernel stack pages */ -- 2.11.0 From c2d432e402309cc5900a3a895afc3483b1c89a50 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 30 May 2015 14:42:43 +0200 Subject: [PATCH 087/257] sys/conf/newvers.sh: Ditch support for p4 (non-free) and hg (not relevant for ElectroBSD) Obtained from: ElectroBSD --- sys/conf/newvers.sh | 49 ++----------------------------------------------- 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 7c9c0474d218..c28bb4aeabba 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -153,11 +153,6 @@ if [ -z "${svnversion}" ] && [ -x /usr/bin/svnliteversion ] ; then fi fi -for dir in /usr/bin /usr/local/bin; do - if [ -x "${dir}/p4" ] && [ -z ${p4_cmd} ] ; then - p4_cmd=${dir}/p4 - fi -done if [ -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then @@ -167,15 +162,6 @@ if [ -d "${SYSDIR}/../.git" ] ; then done fi -if [ -d "${SYSDIR}/../.hg" ] ; then - for dir in /usr/bin /usr/local/bin; do - if [ -x "${dir}/hg" ] ; then - hg_cmd="${dir}/hg -R ${SYSDIR}/.." - break - fi - done -fi - if [ -n "$svnversion" ] ; then svn=`cd ${SYSDIR} && $svnversion 2>/dev/null` case "$svn" in @@ -224,37 +210,6 @@ if [ -n "$git_cmd" ] ; then fi fi -if [ -n "$p4_cmd" ] ; then - p4version=`cd ${SYSDIR} && $p4_cmd changes -m1 "./...#have" 2>&1 | \ - awk '{ print $2 }'` - case "$p4version" in - [0-9]*) - p4version=" ${p4version}" - p4opened=`cd ${SYSDIR} && $p4_cmd opened ./... 2>&1` - case "$p4opened" in - File*) ;; - //*) - p4version="${p4version}+edit" - modified=true - ;; - esac - ;; - *) unset p4version ;; - esac -fi - -if [ -n "$hg_cmd" ] ; then - hg=`$hg_cmd id 2>/dev/null` - svn=`$hg_cmd svn info 2>/dev/null | \ - awk -F': ' '/Revision/ { print $2 }'` - if [ -n "$svn" ] ; then - svn=" r${svn}" - fi - if [ -n "$hg" ] ; then - hg=" ${hg}" - fi -fi - include_metadata=true while getopts rR opt; do case "$opt" in @@ -270,10 +225,10 @@ done shift $((OPTIND - 1)) if [ -z "${include_metadata}" ]; then - VERINFO="${VERSION} ${svn}${git}${hg}${p4version}" + VERINFO="${VERSION} ${svn}${git}" VERSTR="${VERINFO}\\n" else - VERINFO="${VERSION} #${v}${svn}${git}${hg}${p4version}: ${t}" + VERINFO="${VERSION} #${v}${svn}${git}: ${t}" VERSTR="${VERINFO}\\n ${u}@${h}:${d}\\n" fi -- 2.11.0 From da12ee9952674078efda7fa5bb98becde4c6a6ae Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 3 Jun 2015 17:16:10 +0200 Subject: [PATCH 088/257] mandocdb: Normalize inodevs to get reproducible results This is a proof-of-concept that scales poorly due to linear searches. The proper fix is probably to use hash-based lookups and only do it when the user wants reproducible results. I didn't find any automated tests for this so the only testing done is "does 'man' appear to be usable". It indeed appears to be ... Obtained from: ElectroBSD --- contrib/mdocml/mandocdb.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/contrib/mdocml/mandocdb.c b/contrib/mdocml/mandocdb.c index 3b9bda0d612b..5bdbb563c6c5 100644 --- a/contrib/mdocml/mandocdb.c +++ b/contrib/mdocml/mandocdb.c @@ -946,6 +946,34 @@ filescan(const char *file) mlink_add(mlink, &st); } +/* + * Messes up inodevs in a reproducible way as long as + * the call order does not change. The implementation + * is silly and only used as proof of concept. + */ +#define HOPEFULLY_ENOUGH_FOR_EVERYBODY 5000 +static void +normalize_inodev(struct inodev *inodev) { + static size_t table[HOPEFULLY_ENOUGH_FOR_EVERYBODY]; + size_t key; + int i; + + key = inodev->st_ino + inodev->st_dev; + + for (i = 0; i < sizeof(table)/sizeof(table[0]); i++) { + if (table[i] == 0) { + /* New value, add to table*/ + table[i] = key; + } + if (table[i] == key) { + /* Use index as new value */ + inodev->st_ino = i; + inodev->st_dev = i; + return; + } + } +} + static void mlink_add(struct mlink *mlink, const struct stat *st) { @@ -976,6 +1004,7 @@ mlink_add(struct mlink *mlink, const struct stat *st) memset(&inodev, 0, sizeof(inodev)); /* Clear padding. */ inodev.st_ino = st->st_ino; inodev.st_dev = st->st_dev; + normalize_inodev(&inodev); slot = ohash_lookup_memory(&mpages, (char *)&inodev, sizeof(struct inodev), inodev.st_ino); mpage = ohash_find(&mpages, slot); -- 2.11.0 From 741145b8a4a5ab904add83f6df0cbc5432ad18f1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 4 Jun 2015 12:18:16 +0200 Subject: [PATCH 089/257] release/Makefile: Reorder dependencies to reduce the chances that base or kernel parts end up in the src.txz. This is merely a workaround Obtained from: ElectroBSD --- release/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index acebba7924c3..ff0365978dbb 100644 --- a/release/Makefile +++ b/release/Makefile @@ -289,7 +289,7 @@ mini-memstick: mini-memstick.img mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} -packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} +packagesystem: ${EXTRA_PACKAGES} base.txz kernel.txz sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} -- 2.11.0 From a4f5d16ea3a181a880b5d4149329058e1f676508 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 17:30:25 +0200 Subject: [PATCH 090/257] Add strip-freebsd.sh ... which suggests a bunch of stuff to delete from a vanilla FreeBSD checkout. In a previous life it was called free-freebsd.sh which obviously sounds more awesome, but nowadays most of the suggested stuff for removal is actually free software that just isn't relevant for ElectroBSD. Removing code we don't need means we don't have to care about its security and license issues. There's lots of code to remove left! Obtained from: ElectroBSD --- release/scripts/strip-freebsd.sh | 171 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100755 release/scripts/strip-freebsd.sh diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh new file mode 100755 index 000000000000..eaaf3789a0ab --- /dev/null +++ b/release/scripts/strip-freebsd.sh @@ -0,0 +1,171 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## + +# This script pretends to free a FreeBSD checkout from +# known-unfree files and other stuff that is not required +# by ElectroBSD. +# +# While it is pretty much guaranteed to ditch a bunch of files +# it doesn't work very thorougly and the resulting checkout is +# likely to still contain lots of non-free parts that haven't +# been discovered yet. + +get_snd_csa_files() { + find sys/dev/sound/pci -name "csa*" +} + +get_snd_ds1_files() { + find sys/dev/sound/pci -name "ds1*" +} + +get_snd_maestro3_files() { + # This file contains the actual blobs + echo sys/dev/sound/pci/allegro_code.h + + # This file contains the (free) code that relies on the blobs. + echo sys/dev/sound/pci/maestro3.c +} + +# We can't simply remove the whole directory +# as a (free) header is required by bge. +get_bce_files() { + echo "sys/dev/bce/if_bcefw.h" \ + "sys/dev/bce/if_bce.c" +} + +get_usb_firmware_files() { + find sys/dev/usb/ -name "*fw*" +} + +# XXX: Misleading name, some of the files are merely tainted +# by non-free dependencies +get_unfree_files() { + find sys/ -name "*.uu" + get_snd_csa_files + get_snd_ds1_files + get_snd_maestro3_files + get_bce_files + get_usb_firmware_files +} + +get_files_to_ditch() { + get_unfree_files +} + +# These architectures are mainly unsupported by ElectroBSD +# due to lack of hardware for testing purposes. +# +# The source directories are mainly removed to shrink the +# source tarball and to reduce the number of files that +# should be audited for license and security issues. +get_unsupported_architectures() { + echo "arm arm64 mips pc98 powerpc sparc64" +} + +# These depend on or contain proprietary firmware that is included in sys/contrib/dev +get_tainted_sys_contrib_devs() { + echo "drm2 ipw iwi iwm iwn mwl npe otus ral rsu run uath urtwn wpi" +} + +# These require proprietary firmware that is included in sys/dev +# and may cause build failures without it. +get_tainted_sys_devs() { + # bce has already been taken care of by get_bce_files() above + echo "bxe ctau cx cxgb cxgbe ispfw qlxgbe" \ + "spibus it tw" +} + +get_unused_contrib_dirs() { + # XXX: gcc can't be deleted because parts of it are apparently + # required to build libc. This should be investigated more thoroughly, + # hopefully it can be fixed. + echo "apr apr-util ipfilter ofed sendmail serf subversion tcsh" +} + +get_directories_to_ditch() { + local arch \ + dir arch_dir sys_contrib contrib_dir + + for dir in sys sys/boot; do + for arch in $(get_unsupported_architectures); do + potential_directory="${dir}/${arch}" + if [ -d "${potential_directory}" ]; then + echo "${potential_directory}" + fi + done + done + + for sys_contrib in ipfilter octeon-sdk; do + echo "sys/contrib/${sys_contrib}" + done + + for sys_contrib in $(get_tainted_sys_contrib_devs); do + echo "sys/contrib/dev/${sys_contrib}" + done + + for sys_dev in $(get_tainted_sys_devs); do + echo "sys/dev/${sys_dev}" + done + + for contrib_dir in $(get_unused_contrib_dirs); do + potential_directory="contrib/${contrib_dir}" + if [ -d "${potential_directory}" ]; then + echo "${potential_directory}" + fi + done +} + +purify_cwd() { + # There are no spaces in paths or file names. + files_to_ditch="$(get_files_to_ditch)" + for f in $files_to_ditch; do + [ -f "${f}" ] && echo "rm ${f}" + done + + dirs_to_ditch="$(get_directories_to_ditch)" + for d in $dirs_to_ditch; do + [ -d "${d}" ] && echo "rm -r ${d}" + done +} + +main() { + local src_dir \ + files_to_ditch dirs_to_ditch + + src_dir="${1}" + if [ -z "${src_dir}" ]; then + echo "No source directory given" + return 1 + fi + if [ ! -d "${src_dir}" ]; then + echo "No such directory: ${src_dir}" + return 1 + fi + # Make it less likely to operate on a directory + # that isn't actually a FreeBSD checkout + if [ ! -f "${src_dir}/COPYRIGHT" ]; then + echo "${src_dir} contains no COPYRIGHT file" + return 1 + fi + + cd "${src_dir}" || return 1 + + purify_cwd +} + +main "${@}" -- 2.11.0 From 027c459a91b10a8919db13204978f95ec517f7a6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 1 Jan 2016 11:22:49 +0100 Subject: [PATCH 091/257] release/scripts/strip-freebsd.sh: Ditch rtwn which got added in r293009 Obtained from: ElectroBSD --- release/scripts/strip-freebsd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh index eaaf3789a0ab..56be187cca4c 100755 --- a/release/scripts/strip-freebsd.sh +++ b/release/scripts/strip-freebsd.sh @@ -79,7 +79,7 @@ get_unsupported_architectures() { # These depend on or contain proprietary firmware that is included in sys/contrib/dev get_tainted_sys_contrib_devs() { - echo "drm2 ipw iwi iwm iwn mwl npe otus ral rsu run uath urtwn wpi" + echo "drm2 ipw iwi iwm iwn mwl npe otus ral rsu rtwn run uath urtwn wpi" } # These require proprietary firmware that is included in sys/dev -- 2.11.0 From d550032af9ec42c868a0a80dc6115e9392e893a7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Sep 2015 14:33:31 +0200 Subject: [PATCH 092/257] sys/conf/newvers.sh: Remove svn support (but keep 'git svn' support) Obtained from: ElectroBSD --- sys/conf/newvers.sh | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index c28bb4aeabba..8e2b1798228e 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -128,31 +128,6 @@ else fi i=`${MAKE:-make} -V KERN_IDENT` -for dir in /usr/bin /usr/local/bin; do - if [ ! -z "${svnversion}" ] ; then - break - fi - if [ -x "${dir}/svnversion" ] && [ -z ${svnversion} ] ; then - # Run svnversion from ${dir} on this script; if return code - # is not zero, the checkout might not be compatible with the - # svnversion being used. - ${dir}/svnversion $(realpath ${0}) >/dev/null 2>&1 - if [ $? -eq 0 ]; then - svnversion=${dir}/svnversion - break - fi - fi -done - -if [ -z "${svnversion}" ] && [ -x /usr/bin/svnliteversion ] ; then - /usr/bin/svnliteversion $(realpath ${0}) >/dev/null 2>&1 - if [ $? -eq 0 ]; then - svnversion=/usr/bin/svnliteversion - else - svnversion= - fi -fi - if [ -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then @@ -162,22 +137,6 @@ if [ -d "${SYSDIR}/../.git" ] ; then done fi -if [ -n "$svnversion" ] ; then - svn=`cd ${SYSDIR} && $svnversion 2>/dev/null` - case "$svn" in - [0-9]*[MSP]|*:*) - svn=" r${svn}" - modified=true - ;; - [0-9]*) - svn=" r${svn}" - ;; - *) - unset svn - ;; - esac -fi - if [ -n "$git_cmd" ] ; then git=`$git_cmd rev-parse --verify --short HEAD 2>/dev/null` svn=`$git_cmd svn find-rev $git 2>/dev/null` -- 2.11.0 From 9d7257f67edd7dde2bf3c0b478103a83bfc37984 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Sep 2015 14:34:45 +0200 Subject: [PATCH 093/257] sys/conf/newvers.sh: Do not add git hash from .git if KERNEL_VERSION_NUMBER is defined ... as this indicates that we are (trying to) reproduce a build. Do not use REPRO_SEED as it's always set nowadays. XXX: There should be a cleaner way to do this. Obtained from: ElectroBSD --- sys/conf/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 8e2b1798228e..3d2b8774edb5 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -128,7 +128,7 @@ else fi i=`${MAKE:-make} -V KERN_IDENT` -if [ -d "${SYSDIR}/../.git" ] ; then +if [ -z "${KERNEL_VERSION_NUMBER}" -a -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then git_cmd="${dir}/git --git-dir=${SYSDIR}/../.git" -- 2.11.0 From 705c5689049d42fbd709af46b37df9d117e61339 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 28 May 2015 15:04:48 +0200 Subject: [PATCH 094/257] sys/conf/newvers.sh: Allow to overwrite the kernel version ... as the object directory may be dirty. Obtained from: ElectroBSD --- sys/conf/newvers.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 3d2b8774edb5..2562ab031191 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -109,7 +109,7 @@ then fi touch version -v=`cat version` +v=${KERNEL_VERSION_NUMBER:-`cat version`} u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} @@ -205,4 +205,6 @@ int osreldate = ${RELDATE}; char kern_ident[] = "${i}"; EOF -echo $((v + 1)) > version +if [ -z "${KERNEL_VERSION_NUMBER}" ]; then + echo $((v + 1)) > version +fi -- 2.11.0 From 7b67aa3437ca3880e54637a3e51178b388c0a681 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 17 Sep 2015 11:53:45 +0200 Subject: [PATCH 095/257] make-memstick.sh: Additionally fake uid and gid on the created fs Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index a5d88e166e23..25c00a501131 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -30,7 +30,12 @@ fi echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local -mtree -c -k time -p "${1}" | sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${2}.mtree" +# Prepare mtree spec to fake timestamp, owner and group. +# As a result, man pages will be owned by root instead of man. +# Unfortunately we can't simply reuse ${1}/METALOG as it is incomplete. +mtree -c -k time -p "${1}" | sed \ + -e "s@time=.*@time=${EPOCH_DATE-0}.000000000 uname=root gname=wheel@" \ + > "${2}.mtree" || return 1 makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" -- 2.11.0 From 46e003d48ab620e7f248c829380477b9c4b7e1ea Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 30 Sep 2015 13:15:44 +0200 Subject: [PATCH 096/257] release/amd64/make-memstick.sh: Error out if mkimg fails instead of cleaning up Makes debugging more convenient. Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 25c00a501131..a39a200dde22 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -46,5 +46,9 @@ rm ${1}/etc/rc.conf.local rm "${2}.mtree" mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +if [ $? -ne 0 ]; then + echo "mkimg failed" + exit 1 +fi rm ${2}.part -- 2.11.0 From bc7948f3eb1d640735d275562296c57b8c176bc9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Dec 2015 11:43:06 +0100 Subject: [PATCH 097/257] release/amd64/make-memstick.sh: Stop creating a puny swap partion Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index a39a200dde22..226846ff1c8d 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -45,7 +45,7 @@ rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local rm "${2}.mtree" -mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} if [ $? -ne 0 ]; then echo "mkimg failed" exit 1 -- 2.11.0 From 2d9581f515be451ddfa5d8c8da21394b08b0d138 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Dec 2015 11:44:10 +0100 Subject: [PATCH 098/257] release/amd64/make-memstick.sh: Stop creating an EFI partition as the EFI goo does not build reproducible Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 226846ff1c8d..3feded5a1440 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -45,7 +45,7 @@ rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local rm "${2}.mtree" -mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} if [ $? -ne 0 ]; then echo "mkimg failed" exit 1 -- 2.11.0 From 8cc4c2f4786116971d9f77304bcdb749aaa9212e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Dec 2016 12:25:38 +0100 Subject: [PATCH 099/257] Detach telnetd from the build Obtained from: ElectroBSD --- secure/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/secure/Makefile b/secure/Makefile index c3b2c87e1462..98784c830ad8 100644 --- a/secure/Makefile +++ b/secure/Makefile @@ -12,7 +12,7 @@ _tests= tests # These are the programs which depend on crypto, but not Kerberos. SPROGS= lib/libfetch lib/libpam lib/libradius lib/libtelnet \ - bin/ed libexec/telnetd usr.bin/fetch usr.bin/telnet \ + bin/ed usr.bin/fetch usr.bin/telnet \ usr.sbin/pkg_install usr.sbin/ppp usr.sbin/tcpdump/tcpdump .if ${MK_SENDMAIL} != "no" SPROGS+=usr.sbin/sendmail -- 2.11.0 From 1976fc81a47046d10642f2ce168d7382ebe99b4a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 4 Nov 2015 18:55:18 +0100 Subject: [PATCH 100/257] share/doc: Detach 'legal' from the build It (tries to) install proprietary licenses for code that is not part of ElectroBSD and deleted by reproduce.sh's auto-untaint mode (-a). Obtained from: ElectroBSD --- share/doc/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/share/doc/Makefile b/share/doc/Makefile index 7a02b2952598..5b5e507bc130 100644 --- a/share/doc/Makefile +++ b/share/doc/Makefile @@ -5,7 +5,6 @@ SUBDIR= ${_IPv6} \ ${_atf} \ - legal \ ${_llvm} \ ${_pjdfstest} \ ${_roffdocs} -- 2.11.0 From 19922a8843c26cc1113e24a4412e3b7ce2b3461a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 20 May 2015 13:38:21 +0200 Subject: [PATCH 101/257] sys/boot/Makefile.amd64: Detach efi from the built It doesn't built reproducible, among other things due to the embedded file system, and none of the ElectroBSD users I'm aware of have efi-capable systems anyway. Obtained from: ElectroBSD --- sys/boot/Makefile.amd64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/Makefile.amd64 b/sys/boot/Makefile.amd64 index 5e730bd6c62f..f8e173cbb2f3 100644 --- a/sys/boot/Makefile.amd64 +++ b/sys/boot/Makefile.amd64 @@ -1,6 +1,6 @@ # $FreeBSD$ -SUBDIR+= efi +#SUBDIR+= efi SUBDIR+= libstand32 SUBDIR+= zfs SUBDIR+= userboot -- 2.11.0 From dc7e8aa6e6b1c02dc6e7ca0676e916b91fa4dd53 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 21 May 2015 18:40:56 +0200 Subject: [PATCH 102/257] release/amd64/make-memstick.sh: Allow to overwrite VOLUME_LABEL through the environment Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 3feded5a1440..66ee073c63ba 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -12,6 +12,7 @@ PATH=/bin:/usr/bin:/sbin:/usr/sbin export PATH +VOLUME_LABEL=${VOLUME_LABEL-"FreeBSD_Install"} if [ $# -ne 2 ]; then echo "make-memstick.sh /path/to/directory /path/to/image/file" @@ -28,7 +29,7 @@ if [ -e ${2} ]; then exit 1 fi -echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab +echo "/dev/ufs/${VOLUME_LABEL} / ufs ro,noatime 1 1" > ${1}/etc/fstab echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local # Prepare mtree spec to fake timestamp, owner and group. # As a result, man pages will be owned by root instead of man. @@ -36,7 +37,7 @@ echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local mtree -c -k time -p "${1}" | sed \ -e "s@time=.*@time=${EPOCH_DATE-0}.000000000 uname=root gname=wheel@" \ > "${2}.mtree" || return 1 -makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} +makefs -B little -o label="${VOLUME_LABEL}" -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" exit 1 -- 2.11.0 From 1ee4952f785d1c2e6b8ce83afaf01fb4d7615979 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 21 May 2015 18:42:42 +0200 Subject: [PATCH 103/257] release/Makefile: Export VOLUME_LABEL for make-memstick.sh Obtained from: ElectroBSD --- release/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release/Makefile b/release/Makefile index ff0365978dbb..a7f034aa4e0f 100644 --- a/release/Makefile +++ b/release/Makefile @@ -80,6 +80,8 @@ VOLUME_LABEL= ${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-]/_/g}_${TARGET_ARCH} VOLUME_LABEL= FreeBSD_Install .endif +.export VOLUME_LABEL + .if !exists(${DOCDIR}) NODOC= true .endif -- 2.11.0 From 76220b9dc911b8a00518eb1f4b75e83377e17d5b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 3 Dec 2015 18:34:21 +0100 Subject: [PATCH 104/257] lib/libkvm: Unbreak the build without kernel sources that don't ship with ElectroBSD Obtained from: ElectroBSD --- lib/libkvm/Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/libkvm/Makefile b/lib/libkvm/Makefile index dc611a4b4bfe..8a6806102797 100644 --- a/lib/libkvm/Makefile +++ b/lib/libkvm/Makefile @@ -12,13 +12,8 @@ WARNS?= 3 SRCS= kvm.c kvm_cptime.c kvm_getloadavg.c \ kvm_getswapinfo.c kvm_pcpu.c kvm_proc.c kvm_vnet.c \ - kvm_minidump_aarch64.c \ kvm_amd64.c kvm_minidump_amd64.c \ - kvm_arm.c kvm_minidump_arm.c \ - kvm_i386.c kvm_minidump_i386.c \ - kvm_minidump_mips.c \ - kvm_powerpc.c kvm_powerpc64.c \ - kvm_sparc64.c + kvm_i386.c kvm_minidump_i386.c INCS= kvm.h LIBADD= elf -- 2.11.0 From acb5c6eec9d586cb80a34246fe178ca9c1b795ca Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 May 2015 14:03:06 +0200 Subject: [PATCH 105/257] usr.sbin/pkg: Change URL_SCHEME_PREFIX to an URL more easily controlled by the user ... through ssh port forwarding. Obtained from: ElectroBSD --- usr.sbin/pkg/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr.sbin/pkg/config.c b/usr.sbin/pkg/config.c index 2fea11f50fe4..8cc35190054f 100644 --- a/usr.sbin/pkg/config.c +++ b/usr.sbin/pkg/config.c @@ -64,7 +64,7 @@ static struct config_entry c[] = { [PACKAGESITE] = { PKG_CONFIG_STRING, "PACKAGESITE", - URL_SCHEME_PREFIX "http://pkg.FreeBSD.org/${ABI}/latest", + URL_SCHEME_PREFIX "http://127.0.0.1:8000/packages/${ABI}/", NULL, NULL, false, -- 2.11.0 From 9b1c1a5baf575c99dd6359b142b3e8dd54120baa Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 May 2015 15:33:59 +0200 Subject: [PATCH 106/257] share/doc: Exclude directories that contain (partly) non-free materials Obtained from: ElectroBSD --- share/doc/Makefile | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/share/doc/Makefile b/share/doc/Makefile index 5b5e507bc130..dc2003f7ecfe 100644 --- a/share/doc/Makefile +++ b/share/doc/Makefile @@ -3,11 +3,9 @@ .include -SUBDIR= ${_IPv6} \ - ${_atf} \ +SUBDIR= ${_atf} \ ${_llvm} \ - ${_pjdfstest} \ - ${_roffdocs} + ${_pjdfstest} .if ${MK_TESTS} != "no" _atf= atf @@ -18,14 +16,11 @@ _pjdfstest= pjdfstest _llvm= llvm .endif -.if ${MK_INET6} != "no" -_IPv6= IPv6 -.endif - -# FIXME this is not a real solution ... -.if ${MK_GROFF} != "no" -_roffdocs= papers psd smm usd -.endif +# Note: Documents in the sub directories IPv6, papers, psd, smm +# and usd are not installed on ElectroBSD due to license problems. +# +# Some of the excluded papers actually have free licenses and +# should eventually be installed. SUBDIR_PARALLEL= -- 2.11.0 From 33ab62ca2786333d1f0d75b3a097369eafac90ba Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 May 2015 10:10:43 +0200 Subject: [PATCH 107/257] geom: Do not build raid, raid3 and vinum classes Obtained from: ElectroBSD --- sys/modules/geom/Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/sys/modules/geom/Makefile b/sys/modules/geom/Makefile index 8d7e3c6deb3f..fe7e479840aa 100644 --- a/sys/modules/geom/Makefile +++ b/sys/modules/geom/Makefile @@ -16,13 +16,10 @@ SUBDIR= geom_bde \ geom_multipath \ geom_nop \ geom_part \ - geom_raid \ - geom_raid3 \ geom_sched \ geom_shsec \ geom_stripe \ geom_uzip \ - geom_vinum \ geom_virstor \ geom_zero -- 2.11.0 From 693af0d71b079fa9f1907f933bb6324512f8c116 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 May 2015 12:17:36 +0200 Subject: [PATCH 108/257] share/mk/src.opts.mk: Disable a bunch of options by default .. either due to license issues or because they are not considered relevant for the majority of ElectroBSD users. XXX: This commit does not regenerate src.conf(5) as the process hasn't been reverse-engineered yet. Obtained from: ElectroBSD --- share/mk/src.opts.mk | 60 ++++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index 17c0914a1374..fabe6be40fe1 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -49,7 +49,6 @@ __DEFAULT_YES_OPTIONS = \ AMD \ APM \ AT \ - ATM \ AUDIT \ AUTHPF \ AUTOFS \ @@ -57,23 +56,19 @@ __DEFAULT_YES_OPTIONS = \ BINUTILS \ BINUTILS_BOOTSTRAP \ BLACKLIST \ - BLUETOOTH \ BOOT \ BOOTPARAMD \ BOOTPD \ BSD_CPIO \ - BSDINSTALL \ BSNMP \ BZIP2 \ CALENDAR \ CAPSICUM \ CASPER \ - CCD \ CDDL \ CPP \ CROSS_COMPILER \ CRYPT \ - CTM \ CUSE \ CXX \ DICT \ @@ -87,14 +82,9 @@ __DEFAULT_YES_OPTIONS = \ FDT \ FILE \ FINGER \ - FLOPPY \ - FMTREE \ FORTH \ FP_LIBC \ - FREEBSD_UPDATE \ - FTP \ GAMES \ - GCOV \ GDB \ GNU \ GNU_GREP_COMPAT \ @@ -108,9 +98,7 @@ __DEFAULT_YES_OPTIONS = \ INET \ INET6 \ INETD \ - IPFILTER \ IPFW \ - ISCSI \ JAIL \ KDUMP \ KVM \ @@ -129,7 +117,6 @@ __DEFAULT_YES_OPTIONS = \ MAILWRAPPER \ MAKE \ MANDOCDB \ - NDIS \ NETCAT \ NETGRAPH \ NLS_CATALOGS \ @@ -137,36 +124,21 @@ __DEFAULT_YES_OPTIONS = \ NTP \ OPENSSL \ PAM \ - PC_SYSINSTALL \ PF \ PKGBOOTSTRAP \ - PMC \ - PORTSNAP \ - PPP \ QUOTAS \ RADIUS_SUPPORT \ - RCMDS \ - RBOOTD \ RCS \ RESCUE \ ROUTED \ - SENDMAIL \ SETUID_LOGIN \ SHAREDOCS \ - SOURCELESS \ - SOURCELESS_HOST \ - SOURCELESS_UCODE \ - SVNLITE \ SYSCONS \ SYSTEM_COMPILER \ TALK \ - TCP_WRAPPERS \ - TCSH \ TELNET \ TESTS \ TEXTPROC \ - TFTP \ - TIMED \ UNBOUND \ USB \ UTMPX \ @@ -192,6 +164,38 @@ __DEFAULT_NO_OPTIONS = \ SVN \ +# Disable a bunch of additional options that default to yes in FreeBSD +__DEFAULT_NO_OPTIONS += \ + ATM \ + BLUETOOTH \ + BSDINSTALL \ + CCD \ + CTM \ + FLOPPY \ + FMTREE \ + FREEBSD_UPDATE \ + FTP \ + GCOV \ + IPFILTER \ + ISCSI \ + NDIS \ + RBOOTD \ + PC_SYSINSTALL \ + PMC \ + PORTSNAP \ + PPP \ + RCMDS \ + SENDMAIL \ + SVNLITE \ + SOURCELESS \ + SOURCELESS_HOST \ + SOURCELESS_UCODE \ + SYSINSTALL \ + TCP_WRAPPERS \ + TCSH \ + TFTP \ + TIMED \ + # # Default behaviour of some options depends on the architecture. Unfortunately # this means that we have to test TARGET_ARCH (the buildworld case) as well -- 2.11.0 From d3e9eac87ed90db96f59fa09227eb51464e85298 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 24 Oct 2015 12:58:42 +0200 Subject: [PATCH 109/257] share/mk/src.opts.mk: Disable EXAMPLES ... so we don't have to care about license issues. Some of the examples are non-free, in some cases the license is unclear. XXX: Some of the examples are properly licensed and useful, for example share/examples/bhyve/vmrun.sh. It can be executed from a checkout directly, though. Obtained from: ElectroBSD --- share/mk/src.opts.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index fabe6be40fe1..5393b3cab97f 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -78,7 +78,6 @@ __DEFAULT_YES_OPTIONS = \ EE \ ELFCOPY_AS_OBJCOPY \ ELFTOOLCHAIN_BOOTSTRAP \ - EXAMPLES \ FDT \ FILE \ FINGER \ @@ -171,6 +170,7 @@ __DEFAULT_NO_OPTIONS += \ BSDINSTALL \ CCD \ CTM \ + EXAMPLES \ FLOPPY \ FMTREE \ FREEBSD_UPDATE \ -- 2.11.0 From 4731a757561153c55ccac1f4baef670e21ceb6a2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 24 Jun 2015 12:53:20 +0200 Subject: [PATCH 110/257] share/mk/src.opts.mk: Enable CLANG_EXTRAS by default as it's required for llvm-symbolizer Obtained from: ElectroBSD --- share/mk/src.opts.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index 5393b3cab97f..494cdf976848 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -150,7 +150,6 @@ __DEFAULT_YES_OPTIONS = \ __DEFAULT_NO_OPTIONS = \ BSD_GREP \ - CLANG_EXTRAS \ DTRACE_TESTS \ EISA \ HESIOD \ @@ -163,6 +162,10 @@ __DEFAULT_NO_OPTIONS = \ SVN \ +# Enable additional options that default to NO in FreeBSD +__DEFAULT_YES_OPTIONS += \ + CLANG_EXTRAS \ + # Disable a bunch of additional options that default to yes in FreeBSD __DEFAULT_NO_OPTIONS += \ ATM \ -- 2.11.0 From c8aafab7a77dfbc46716f5475010d7e515aaebd7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 May 2015 17:53:19 +0200 Subject: [PATCH 111/257] etc/master.passwd: Change root's login shell to sh Now that csh is no longer compiled by default that seems like a rather swell idea. Obtained from: ElectroBSD --- etc/master.passwd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/master.passwd b/etc/master.passwd index 689b64e78473..c6005dc66899 100644 --- a/etc/master.passwd +++ b/etc/master.passwd @@ -1,6 +1,6 @@ # $FreeBSD$ # -root::0:0::0:0:Charlie &:/root:/bin/csh +root::0:0::0:0:Charlie &:/root:/bin/sh toor:*:0:0::0:0:Bourne-again Superuser:/root: daemon:*:1:1::0:0:Owner of many system processes:/root:/usr/sbin/nologin operator:*:2:5::0:0:System &:/:/usr/sbin/nologin -- 2.11.0 From 31d0f450f043866341d7d94b46973d4c8eea423f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 5 Jun 2015 12:03:46 +0200 Subject: [PATCH 112/257] release/Makefile: Set German keyboard map Obtained from: ElectroBSD --- release/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/release/Makefile b/release/Makefile index a7f034aa4e0f..7f2e8de5c721 100644 --- a/release/Makefile +++ b/release/Makefile @@ -219,6 +219,7 @@ disc1: packagesystem ln -fs /tmp/bsdinstall_etc/resolv.conf ${.TARGET}/etc/resolv.conf echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf + echo keymap=\"de\" >> ${.TARGET}/etc/rc.conf echo vfs.mountroot.timeout=\"10\" >> ${.TARGET}/boot/loader.conf cp ${.CURDIR}/rc.local ${.TARGET}/etc touch ${.TARGET} -- 2.11.0 From d4d705e64973f2d5c648cf7bac4518b4541d7061 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 5 Jun 2015 12:06:14 +0200 Subject: [PATCH 113/257] release/Makefile: Set hostname to ${VOLUME_LABEL} Obtained from: ElectroBSD --- release/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/release/Makefile b/release/Makefile index 7f2e8de5c721..a28ce1e98115 100644 --- a/release/Makefile +++ b/release/Makefile @@ -220,6 +220,7 @@ disc1: packagesystem echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf echo keymap=\"de\" >> ${.TARGET}/etc/rc.conf + echo hostname=\"${VOLUME_LABEL}\" >> ${.TARGET}/etc/rc.conf echo vfs.mountroot.timeout=\"10\" >> ${.TARGET}/boot/loader.conf cp ${.CURDIR}/rc.local ${.TARGET}/etc touch ${.TARGET} -- 2.11.0 From de88f2958712b53c5357db4829d6864a851168da Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 5 Jun 2015 19:38:39 +0200 Subject: [PATCH 114/257] Stop bothering packaging docs which are no longer built. Squash or relocate Obtained from: ElectroBSD --- Makefile.inc1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index c27fefbe8ad1..77b39df7f70a 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -922,7 +922,7 @@ ITOOLS+=makewhatis # # Non-base distributions produced by the base system -EXTRA_DISTRIBUTIONS= doc +EXTRA_DISTRIBUTIONS= .if defined(LIBCOMPAT) EXTRA_DISTRIBUTIONS+= lib${libcompat} .endif -- 2.11.0 From 13ba50f273a4355a2e818d5866c32ea0b0eec11b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 10 Jun 2015 12:29:37 +0200 Subject: [PATCH 115/257] release/Makefile: Add ${DIST_TARBALL_DIR} ... which contains the distribution tarball directory on the install media. Obtained from: ElectroBSD --- release/Makefile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/release/Makefile b/release/Makefile index a28ce1e98115..bc3b6076118e 100644 --- a/release/Makefile +++ b/release/Makefile @@ -60,6 +60,9 @@ DISTDIR= dist NO_ROOT=1 .export NO_ROOT +# Path to the distribution tarballs on the created installation media. +DIST_TARBALL_DIR=usr/freebsd-dist + # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) .for _V in TYPE BRANCH REVISION @@ -207,9 +210,9 @@ disc1: packagesystem MK_DEBUG_FILES=no MK_LLDB=no \ MK_TOOLCHAIN=no # Copy distfiles - mkdir -p ${.TARGET}/usr/freebsd-dist + mkdir -p ${.TARGET}/${DIST_TARBALL_DIR} for dist in MANIFEST $$(ls *.txz | grep -vE -- '(base|lib32|kernel)-dbg'); \ - do cp $${dist} ${.TARGET}/usr/freebsd-dist; \ + do cp $${dist} ${.TARGET}/${DIST_TARBALL_DIR}; \ done # Copy documentation, if generated .if !defined(NODOC) @@ -236,8 +239,8 @@ bootonly: packagesystem MK_RESCUE=no MK_DICT=no \ MK_KERNEL_SYMBOLS=no MK_TESTS=no MK_DEBUG_FILES=no # Copy manifest only (no distfiles) to get checksums - mkdir -p ${.TARGET}/usr/freebsd-dist - cp MANIFEST ${.TARGET}/usr/freebsd-dist + mkdir -p ${.TARGET}/${DIST_TARBALL_DIR} + cp MANIFEST ${.TARGET}/${DIST_TARBALL_DIR} # Copy documentation, if generated .if !defined(NODOC) cp reldoc/* ${.TARGET} @@ -256,9 +259,9 @@ dvd: packagesystem DESTDIR=${.OBJDIR}/${.TARGET} MK_RESCUE=no MK_KERNEL_SYMBOLS=no \ MK_TESTS=no MK_DEBUG_FILES=no # Copy distfiles - mkdir -p ${.TARGET}/usr/freebsd-dist + mkdir -p ${.TARGET}/${DIST_TARBALL_DIR} for dist in MANIFEST $$(ls *.txz | grep -v -- '(base|lib32)-dbg'); \ - do cp $${dist} ${.TARGET}/usr/freebsd-dist; \ + do cp $${dist} ${.TARGET}/${DIST_TARBALL_DIR}; \ done # Copy documentation, if generated .if !defined(NODOC) -- 2.11.0 From 7357375269c684d770aa5713fb72e86970c6d909 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 10 Jun 2015 12:36:37 +0200 Subject: [PATCH 116/257] release/Makefile: ElectroBSDify the shiny new ${DIST_TARBALL_DIR} Obtained from: ElectroBSD --- release/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index bc3b6076118e..1ed856579a1c 100644 --- a/release/Makefile +++ b/release/Makefile @@ -61,7 +61,7 @@ NO_ROOT=1 .export NO_ROOT # Path to the distribution tarballs on the created installation media. -DIST_TARBALL_DIR=usr/freebsd-dist +DIST_TARBALL_DIR=usr/electrobsd-dist # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) -- 2.11.0 From e31320a3eda0167ac3d0350540eff780441fa183 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 15 Jun 2015 12:57:51 +0200 Subject: [PATCH 117/257] Add rc.d script to automatically enable soft-protection on boot Obtained from: ElectroBSD --- etc/rc.d/Makefile | 1 + etc/rc.d/soft-protection | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 etc/rc.d/soft-protection diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index 320e550aeb8a..ec3ad74a4bae 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -104,6 +104,7 @@ FILES= DAEMON \ savecore \ securelevel \ serial \ + soft-protection \ sppp \ statd \ static_arp \ diff --git a/etc/rc.d/soft-protection b/etc/rc.d/soft-protection new file mode 100755 index 000000000000..58bc02a0c037 --- /dev/null +++ b/etc/rc.d/soft-protection @@ -0,0 +1,46 @@ +#!/bin/sh +# +########################################################################### +# +# soft-protection - Enables cloudiatr soft protection on boot +# +########################################################################### +# +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR +# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT +# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. +########################################################################### + +# PROVIDE: soft_protection +# REQUIRE: FILESYSTEMS + +. /etc/rc.subr + +name="soft_protection" +rcvar="soft_protection_enable" + +soft_protection_enable="${soft_protection_enable-NO}" + +start_cmd="enable_soft_protection" +stop_cmd=":" + +enable_soft_protection() { + # We don't use the soft-protect subcommand + # because it may require user feedback. + cloudiatr cmd cloudiatr_soft_protect +} + +load_rc_config "${name}" +run_rc_command "${1}" -- 2.11.0 From 31cde9b376ba81f65ae91e189a9775e34084a785 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 16 Jun 2015 15:21:46 +0200 Subject: [PATCH 118/257] Import geli-key-monitor Obtained from: ElectroBSD --- share/dtrace/Makefile | 1 + share/dtrace/README | 3 +- share/dtrace/geli-key-monitor | 191 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100755 share/dtrace/geli-key-monitor diff --git a/share/dtrace/Makefile b/share/dtrace/Makefile index efd8b08706ba..742fa6ea1ab7 100644 --- a/share/dtrace/Makefile +++ b/share/dtrace/Makefile @@ -11,6 +11,7 @@ SCRIPTS= blocking \ disklatencycmd \ hotopen \ nfsattrstats \ + geli-key-monitor \ nfsclienttime \ siftr \ tcpconn \ diff --git a/share/dtrace/README b/share/dtrace/README index 6855c6aee721..aeb2a06727b5 100644 --- a/share/dtrace/README +++ b/share/dtrace/README @@ -2,4 +2,5 @@ $FreeBSD$ This directory contains scripts for use with the DTrace system. These files and directories contain code generated by the FreeBSD -Project for use with DTrace on FreeBSD. +Project for use with DTrace on FreeBSD and code generated by the +ElectroBSD project for use with DTrace on ElectroBSD. diff --git a/share/dtrace/geli-key-monitor b/share/dtrace/geli-key-monitor new file mode 100755 index 000000000000..2cf4ba5c09be --- /dev/null +++ b/share/dtrace/geli-key-monitor @@ -0,0 +1,191 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * geli-key-monitor + * + * Traces GELI to print the beginning of various keys and warns + * about already known ones. For motivation and example output see: + * https://www.fabiankeil.de/gehacktes/geli-key-monitor/ + * + * Copyright (c) 2012 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + algo[ 2] = "3DES-CBC"; + algo[ 3] = "Blowfish-CBC"; + algo[11] = "AES-CBC"; + algo[21] = "Camellia-CBC"; + algo[22] = "AES-XTS"; + + /* + * These are the first bytes of known weak keys generated with + * an unitialized sc->sc_ekey on a little-endian system. + */ + known_keys[0x06d800ef] = 1; + known_keys[0x252c6a6d] = 1; + known_keys[0xa72b9c7c] = 1; + known_keys[0xfde44683] = 1; + + G_ELI_FLAG_ENC_IVKEY = 0x00400000; + + g_eli_key_stats_available = 0; + g_eli_hold_stats_available = 0; + + /* + * Geli v5 and higher use a different key for each GB, + * so large disks have several thousand keys. + * Monitoring only the first few should do, keeps the + * output and reduces the likelihood of prevents dynvar + * drops. + */ + max_disk_keys_to_monitor = 4; + + printf("%Y: Monitoring geli keys (up to %d encryption keys per disk). %s\n", + walltimestamp, max_disk_keys_to_monitor, "Press CTRL-C to exit."); +} + +fbt::g_eli_mkey_propagate:entry +{ + self->sc = (struct g_eli_softc *)arg0; + self->mkey = (struct g_eli_key *)arg1; + + self->geom_name = stringof(self->sc->sc_geom->name); + self->monitored_keys = 0; + self->key_limit_reached = 0; + + this->algo = self->sc->sc_ealgo; + this->algo_name = (algo[this->algo] != NULL) ? algo[this->algo] : "Unregistered"; + this->geli_version = self->sc->sc_version; + + printf("%Y: %s: %s:%-6s: version: %d, algo: %s, flags: 0x%x (FLAG_ENC_IVKEY: %d) ", + walltimestamp, + self->geom_name, + probefunc, probename, + this->geli_version, this->algo_name, + self->sc->sc_flags, + (self->sc->sc_flags & G_ELI_FLAG_ENC_IVKEY) != 0 + ); + printf("mkey: %08.8x, sc_mkey: %08.8x, sc_ekey: %08.8x\n", + *(uint32_t *)(self->mkey), + *(uint32_t *)(self->sc->sc_mkey), + *(uint32_t *)(self->sc->sc_ekey)); +} + +fbt::g_eli_mkey_propagate:return +/self->sc != NULL/ +{ + printf("%Y: %s: %s:%-6s: mkey: %08.8x, sc_mkey: %08.8x, sc_ekey: %08.8x, sc_ekeys_allocated: %d\n", + walltimestamp, + self->geom_name, + probefunc, probename, + *(uint32_t *)(self->mkey), + *(uint32_t *)(self->sc->sc_mkey), + *(uint32_t *)(self->sc->sc_ekey), + self->sc->sc_ekeys_allocated); +} + +fbt::g_eli_key_fill:entry +{ + self->sc = (struct g_eli_softc *)arg0; + self->key = (struct g_eli_key *)arg1; + self->geom_name = stringof(self->sc->sc_geom->name); +} + +fbt::g_eli_key_fill: +/(self->sc != NULL) && !self->key_limit_reached/ +{ + this->key = *(uint32_t *)(self->key->gek_key); + printf("%Y: %s: %s:%-6s: key->gek_key: %8.8x (%d).%s\n", + walltimestamp, + self->geom_name, + probefunc, probename, + this->key, + self->monitored_keys, + known_keys[this->key] ? " Key looks familiar!" : ""); +} + +fbt::g_eli_key_fill:return +/(self->sc != NULL) && !self->key_limit_reached/ +{ + this->key = *(uint32_t *)(self->key->gek_key); + @g_eli_keys[self->monitored_keys, this->key, self->geom_name] = count(); + /* + * Register the generated key as known. + * Unless the provider is reattached we do not want to see it again. + */ + known_keys[this->key] = 1; + g_eli_key_stats_available = 1; + self->monitored_keys++; +} + +fbt::g_eli_key_fill:return +/(self->sc != NULL) && (self->sc->sc_ekeys_allocated == max_disk_keys_to_monitor - 1)/ +{ + printf("%Y: %s: %s:%-6s: Encryption key limit per disk reached.\n", + walltimestamp, self->geom_name, probefunc, probename); + self->key_limit_reached = 1; +} + +fbt::g_eli_key_hold:entry +{ + self->sc = (struct g_eli_softc *)arg0; + self->offset = (off_t)arg1; + self->blocksize = (size_t)arg2; + self->geom_name = stringof(self->sc->sc_geom->name); + + this->first_mkey_bytes = *(uint32_t *)(self->sc->sc_mkey); + this->first_ekey_bytes = *(uint32_t *)(self->sc->sc_ekey); + /* + * We only check the first bytes here, so false-positives are + * theoretically posible although unlikely. + */ + this->empty_ekey = (0 == this->first_ekey_bytes); + + this->geli_version = self->sc->sc_version; + this->algo = self->sc->sc_ealgo; + this->algo_name = (algo[this->algo] != NULL) ? algo[this->algo] : "Unregistered"; + + @g_eli_key_hold[self->geom_name, + this->geli_version, + this->algo_name, + this->algo, + this->first_mkey_bytes, + this->first_ekey_bytes] = count(); + + g_eli_hold_stats_available = 1; +} + +tick-60sec, +dtrace:::END +/g_eli_hold_stats_available/ +{ + printf("\n---------------------------------------------------------\n"); + printf("%Y: g_eli_key_hold() calls so far:\n", walltimestamp); + printf("%-20s %12s %18s %11s %11s %10s\n", + "Provider", "Geli version", "Algorithm", "mkey start", "ekey start", "calls"); + printa("%-20s %12d %13s (%2d) %08x %08x %@10d\n", @g_eli_key_hold); +} + +tick-60sec, +dtrace:::END +/g_eli_key_stats_available/ +{ + printf("\n%Y: g_eli_key count (only works for geli version 5 or higher):\n", walltimestamp); + printa("%@u #%04d %8.8x %-25s\n", @g_eli_keys); +} -- 2.11.0 From 5943491b7b06604d17b2b0c5dcaebe4b94104fcf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 18 Jun 2015 12:52:20 +0200 Subject: [PATCH 119/257] jemalloc: Enable MALLOC_PRODUCTION Obtained from: ElectroBSD --- contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h b/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h index e2ddfd8b9c1c..750d669ca27a 100644 --- a/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h +++ b/contrib/jemalloc/include/jemalloc/jemalloc_FreeBSD.h @@ -4,6 +4,8 @@ #undef JEMALLOC_OVERRIDE_VALLOC +#define MALLOC_PRODUCTION + #ifndef MALLOC_PRODUCTION #define MALLOC_PRODUCTION #endif -- 2.11.0 From c50b4374175c7b7d4dfee17964ce33da5cc23e83 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 16:46:35 +0200 Subject: [PATCH 120/257] release/Makefile: Note that the release process may silently fail Obtained from: ElectroBSD --- release/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/release/Makefile b/release/Makefile index 1ed856579a1c..e4eca6c74f42 100644 --- a/release/Makefile +++ b/release/Makefile @@ -2,6 +2,14 @@ # # Makefile for building releases and release media. # +# Note that the release process is rather fragile and lots +# of variables depend on each other in mysterious and +# undocumented ways. +# +# If you don't get the magic right you should consider yourself +# lucky if the build fails, the usual outcome is a release that +# does not work. +# # User-driven targets: # cdrom: Builds release CD-ROM media (disc1.iso) # dvdrom: Builds release DVD-ROM media (dvd1.iso) -- 2.11.0 From 5920b1a44509ff43e4540e5ef75bce7f777f6b59 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 16:50:54 +0200 Subject: [PATCH 121/257] release/Makefile: Remove more stuff that does not apply to ElecroBSD from the disc1 target Obtained from: ElectroBSD --- release/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index e4eca6c74f42..48c7475c23b5 100644 --- a/release/Makefile +++ b/release/Makefile @@ -227,7 +227,6 @@ disc1: packagesystem cp reldoc/* ${.TARGET} .endif # Set up installation environment - ln -fs /tmp/bsdinstall_etc/resolv.conf ${.TARGET}/etc/resolv.conf echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf echo keymap=\"de\" >> ${.TARGET}/etc/rc.conf -- 2.11.0 From 0a171e8372a01aedc8216937e6dd35ebb7bae38d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 17:08:38 +0200 Subject: [PATCH 122/257] Makefile.inc1: Workaround passwd and friends missing in the base.txz Obtained from: ElectroBSD --- Makefile.inc1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile.inc1 b/Makefile.inc1 index 77b39df7f70a..df8c415be268 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1027,6 +1027,9 @@ distributeworld installworld stageworld: _installcheck_world .PHONY -e 's@(type=)@uid=0 gid=0 time=${EPOCH_DATE}.0 \1@' \ -e 's@//@/@g' -i '.bak' ${METALOG} .endif + @# Workaround for parts of etc mysteriously not being added below base. + @# As usual the location in dist is fine. + sed -e 's@^\./etc@./base/etc@' -i '.etc.bak' ${METALOG} .for dist in base ${EXTRA_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that -- 2.11.0 From ae09cf80f3050a1952c489159a3ec58f4ade2d9d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 27 Jun 2015 19:16:04 +0200 Subject: [PATCH 123/257] ata_da: Ditch support for legacy device names ... and remove the annoying "Previously was known as adX" messages. (The code no longer matches the commit message precisely as upstream had a similar idea a while later and just missed a block which prevented the commit from comletely disappearing while rebasing) Obtained from: ElectroBSD --- sys/cam/ata/ata_da.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c index 808dc72afcb3..6bcc3dba8632 100644 --- a/sys/cam/ata/ata_da.c +++ b/sys/cam/ata/ata_da.c @@ -758,10 +758,6 @@ static void adashutdown(void *arg, int howto); static void adasuspend(void *arg); static void adaresume(void *arg); -#ifndef ADA_DEFAULT_LEGACY_ALIASES -#define ADA_DEFAULT_LEGACY_ALIASES 1 -#endif - #ifndef ADA_DEFAULT_TIMEOUT #define ADA_DEFAULT_TIMEOUT 30 /* Timeout in seconds */ #endif -- 2.11.0 From 5d4bb7b94d9ad4c097a2cd236b55f88c7b380014 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 28 Jun 2015 13:45:09 +0200 Subject: [PATCH 124/257] Use make-memstick.sh's amd64 version on i386 as well Obtained from: ElectroBSD --- release/i386/make-memstick.sh | 44 +------------------------------------------ 1 file changed, 1 insertion(+), 43 deletions(-) mode change 100755 => 120000 release/i386/make-memstick.sh diff --git a/release/i386/make-memstick.sh b/release/i386/make-memstick.sh deleted file mode 100755 index f824382ec4b0..000000000000 --- a/release/i386/make-memstick.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# -# This script generates a "memstick image" (image that can be copied to a -# USB memory stick) from a directory tree. Note that the script does not -# clean up after itself very well for error conditions on purpose so the -# problem can be diagnosed (full filesystem most likely but ...). -# -# Usage: make-memstick.sh -# -# $FreeBSD$ -# - -PATH=/bin:/usr/bin:/sbin:/usr/sbin -export PATH - -if [ $# -ne 2 ]; then - echo "make-memstick.sh /path/to/directory /path/to/image/file" - exit 1 -fi - -if [ ! -d ${1} ]; then - echo "${1} must be a directory" - exit 1 -fi - -if [ -e ${2} ]; then - echo "won't overwrite ${2}" - exit 1 -fi - -echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab -echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local -makefs -B little -o label=FreeBSD_Install ${2}.part ${1} -if [ $? -ne 0 ]; then - echo "makefs failed" - exit 1 -fi -rm ${1}/etc/fstab -rm ${1}/etc/rc.conf.local - -mkimg -s gpt -b ${1}/boot/pmbr -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} -rm ${2}.part - diff --git a/release/i386/make-memstick.sh b/release/i386/make-memstick.sh new file mode 120000 index 000000000000..978e04a02184 --- /dev/null +++ b/release/i386/make-memstick.sh @@ -0,0 +1 @@ +../amd64/make-memstick.sh \ No newline at end of file -- 2.11.0 From 2e73f2ff277a1a7335c86333b09be9f8d350215d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 09:50:52 +0200 Subject: [PATCH 125/257] copyright.h: Use more insightful trademark information Obtained from: ElectroBSD --- sys/sys/copyright.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/sys/copyright.h b/sys/sys/copyright.h index 938cbf02d374..c85ee917ea3c 100644 --- a/sys/sys/copyright.h +++ b/sys/sys/copyright.h @@ -38,7 +38,7 @@ /* Foundation */ #define TRADEMARK_Foundation \ - "FreeBSD is a registered trademark of The FreeBSD Foundation.\n" + "ElectroBSD ain't no registered trademark of The ElectroBSD Foundation (which does not exist).\n" /* Berkeley */ #define COPYRIGHT_UCB \ -- 2.11.0 From c9c1ed3733a808fc342919b80d4547a266278112 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 09:54:11 +0200 Subject: [PATCH 126/257] g_new_provider_event(): Return early if the provider is already withering Previously debug kernels would panic, other side effects haven't been diagnosed yet. Patch by Scott M. Ferris. FreeBSD bug: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=200740 Obtained from: ElectroBSD --- sys/geom/geom_subr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/geom/geom_subr.c b/sys/geom/geom_subr.c index d0df840945c4..7d19a6fbee15 100644 --- a/sys/geom/geom_subr.c +++ b/sys/geom/geom_subr.c @@ -531,8 +531,8 @@ g_new_provider_event(void *arg, int flag) return; pp = arg; G_VALID_PROVIDER(pp); - KASSERT(!(pp->flags & G_PF_WITHER), - ("g_new_provider_event but withered")); + if ((pp->flags & G_PF_WITHER) != 0) + return; LIST_FOREACH_SAFE(cp, &pp->consumers, consumers, next_cp) { if ((cp->flags & G_CF_ORPHAN) == 0 && cp->geom->attrchanged != NULL) -- 2.11.0 From cf8a1e698151bc3306a5dec8863b77f1bd5e5774 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 8 Jul 2015 17:43:31 +0200 Subject: [PATCH 127/257] sys/kern: Remove another FreeBSD reference from the boot messages Obtained from: ElectroBSD --- sys/kern/subr_smp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index 2d1b8be37aaa..c047115feae3 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -157,7 +157,7 @@ mp_start(void *dummy) } cpu_mp_start(); - printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n", + printf("SMP: Multiprocessor System Detected: %d CPUs\n", mp_ncpus); cpu_mp_announce(); } -- 2.11.0 From 1daa19bbba1b9cb6776a93f80d1eb9d498024073 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 12 Jul 2015 13:11:28 +0200 Subject: [PATCH 128/257] tools/test/devrandom: Import arctest d9a5fc80, a wrapper around dieharder Obtained from: ElectroBSD --- tools/test/devrandom/arc4test | 278 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100755 tools/test/devrandom/arc4test diff --git a/tools/test/devrandom/arc4test b/tools/test/devrandom/arc4test new file mode 100755 index 000000000000..c824997a4a38 --- /dev/null +++ b/tools/test/devrandom/arc4test @@ -0,0 +1,278 @@ +#!/bin/sh + +############################################################################ +# arc4test +# +# Collects "entropy" and lets dieharder analyze it later on. The collected +# entropy is split into smaller files so the data collected in multiple +# runs can be easily interleaved and tested together. +# +# By default, entropy files are generated with arc4cat, a wrapper around +# arc4random_buf(3) which is suspected of "not returning very random data" +# between FreeBSD r273872 and r278907. +# +# So far it looks like the data may be "random enough" to pass the tests. +# +# Usage: +# arc4test build : Build arc4cat in $ARC4CAT_DIR +# arc4test collect : Collect potential entropy with arc4cat +# arc4test collect -d : Collect potential entropy with Dilbert PNRG +# arc4test analyze : Interleave collected entropy files and +# pipe them into dieharder. +# arc4test analyze -f : Try to spead up things by caching the interleaved +# entropy in a single file. Reuses the file if it +# already exists. +# arc4test remix : (Re)build an entropy cache file based on the +# previously collected entropy files. Roughfly +# doubles the required disk space but may significantly +# improve performance. +# arc4test cat : Dump interleaved entropy files to stdout +# +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +############################################################################ + +ARC4CAT_DIR=. +ARC4CAT="${ARC4CAT_DIR}/arc4cat" +# May not contain spaces etc. +ENTROPY_DIR="./entropy" +ENTROPY_SUBDIR_PREFIX="collection-" +# This is currently the block size for dd, setting it "too high" +# is not expected to work, however that's not good for input mixing +# later on anyway. +# +# Currently the entropy is split while it's being collected which is +# convenient from a programming point of view, but if the file size +# is small the interleave performance "may" (vulgo: will) suck. +ENTROPY_FILE_SIZE=4k +# Number of entropy files collected per run. If you increase ENTROPY_FILE_SIZE +# (or intend to do lots of collection runs) you may want to decrease this. +MAX_ENTROPY_FILES=100000 +ENTROPY_CACHE_FILE="${ENTROPY_DIR}/cached-entropy" + +prepare() { +} + +# The default entropy source +get_arc4cat_entropy() { + "${ARC4CAT}" +} + +# This is the reverse engineered PRNG from Dilbert strip 2001-10-25: +# http://dilbert.com/strip/2001-10-25 +# +# It is used instead of get_arc4cat_entropy() if the +# collect flag -d is set. +# +# According to the literature (see URL above) you can never be sure +# if the output is random, however the generator seems to fail all +# the dieharder tests and thus doesn't look nearly as good as Yarrow +# and Fortuna. +# +# Until this changes it will not be considered for ElectroBSD. +get_dilbert_entropy() { + while true; do + # The loop has been partially unrolled + # for increased performance! + echo -n "999999" + done +} + +get_shiny_new_entropy() { + local entropy_flag="${1}" + + if [ "${entropy_flag}" = "-d" ]; then + get_dilbert_entropy + else + get_arc4cat_entropy + fi +} + +collect_entropy() { + local entropy_flag \ + i entropy_file entropy_subdir + + entropy_flag="${1}" + i=0 + entropy_subdir="${ENTROPY_DIR}/${ENTROPY_SUBDIR_PREFIX}$(date +%s)" + + mkdir -p "${entropy_subdir}" + + ENTROPY_FILE_POSTFIX="" + + echo "Collecting ${MAX_ENTROPY_FILES} entropy files of size ${ENTROPY_FILE_SIZE} ..." + + # We don't call get_shiny_new_entropy() inside the loop as it + # would result in bits of entropy getting dropped on the floor + # between files. While we don't care about the "waste", we do + # care about not being able to test those bits later on. + get_shiny_new_entropy "${entropy_flag}" | while [ "${i}" -lt "${MAX_ENTROPY_FILES}" ]; do + entropy_file="$(printf "${entropy_subdir}/%.6i" "${i}")" + #echo "Creating ${entropy_file}" + dd bs="${ENTROPY_FILE_SIZE}" count=1 of="${entropy_file}" 2>/dev/null + i=$((i + 1)) + done +} + +create_entropy_cache() { + local \ + entropy_file + + entropy_file="${ENTROPY_DIR}/${LARGE_ENTROPY_FILE_NAME}" + + echo "Building a single entropy file '${ENTROPY_CACHE_FILE}' based on the files collected previously ..." 1>&2 + cat_collected_entropy > "${ENTROPY_CACHE_FILE}" +} + +replay_entropy() { + local fast_flag \ + entropy_file + + fast_flag="${1}" + + if [ "${fast_flag}" = "-f" ]; then + entropy_file="${ENTROPY_DIR}/${LARGE_ENTROPY_FILE_NAME}" + + if [ ! -f "${ENTROPY_CACHE_FILE}" ]; then + create_entropy_cache + fi + cat_entropy_cache + else + cat_collected_entropy + fi +} + +warn_about_entropy_reuse() { + echo "$0: Oh noes, we're out of collected entropy. Going back to the beginning." 1>&2 + echo "This shouldn't be a problem as long as no single test sees repeated data." 1>&2 +} + +cat_entropy_cache() { + while true; do + cat "${ENTROPY_CACHE_FILE}" + warn_about_entropy_reuse + done +} + +cat_collected_entropy() { + local \ + i f entropy_collections entropy_subdir entropy_file + + # XXX: Too fucking slow + #entropy_collections="$(find "${ENTROPY_DIR}/" -name "${ENTROPY_SUBDIR_PREFIX}*" -depth 1 -type 1)" + + # Not best practice but at least the performance doesn't suck + # and it works as expected. + + entropy_collections="${ENTROPY_DIR}/${ENTROPY_SUBDIR_PREFIX}"* + i=0 + while true; do + f="$(printf "%.6i" "${i}")" + for entropy_subdir in $entropy_collections; do + entropy_file="${entropy_subdir}/${f}" + #echo "Catting ${entropy_file}" + cat "${entropy_file}" || return 1 + done + i=$((i + 1)) + if [ "${i}" -eq "${MAX_ENTROPY_FILES}" ]; then + warn_about_entropy_reuse + i=0 + fi + done +} + +get_dieharder_tests() { + dieharder -l | awk '/-d/ {print $2}' +} + +analyze_collected_entropy() { + local fast_flag \ + test_number + + fast_flag="${1}" + + # We call replay_entropy() inside the loop to make sure + # the beginning of the collected entropy is checked by all tests + # (instead of having each test start at different offsets). + for test_number in $(get_dieharder_tests); do + replay_entropy ${fast_flag} | dieharder -g 200 -d "${test_number}" + done +} + +get_arc4cat_code() { + cat< +#include +#include + +int main(void) { + char buf[4096]; + + while (1) { + arc4random_buf(buf, sizeof(buf)); + write(1, buf, sizeof(buf)); + } +} +EOF +} + +build_arc4cat() { + mkdir -p "${ARC4CAT_DIR}" + cd "${ARC4CAT_DIR}" + get_arc4cat_code > arc4cat.c + make arc4cat + rm arc4cat.c +} + +usage() { + echo "Looks like you are doing it wrong. Try one of these:" + echo + echo "$0 analyze" + echo "$0 build" + echo "$0 cat" + echo "$0 collect" + return 1 +} + +main() { + local mode="${1}" + + shift + set -e + prepare + + case "${mode}" in + analyze) + analyze_collected_entropy "${@}" + ;; + build) + build_arc4cat + ;; + cat) + cat_collected_entropy + ;; + collect) + collect_entropy "${@}" + ;; + remix) + create_entropy_cache + ;; + *) + usage + ;; + esac +} + +main "${@}" -- 2.11.0 From b2038fe8c95788de71f331516459d9650b6ed53b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 23 Jul 2015 19:16:44 +0200 Subject: [PATCH 129/257] dtrace_consume(): Warn if cpu cores aren't completely in sync ... instead of aborting. While the results may be not completely accurate, in some cases it may not matter. Previously the assertion would sometimes be triggered on a systems with poor timecounters (TSC-low(-100) ACPI-fast(900) i8254(0) dummy(-1000000)). Obtained from: ElectroBSD --- cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c index 04c022b7ad00..39530cf41e83 100644 --- a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c +++ b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -3325,7 +3326,13 @@ dtrace_consume(dtrace_hdl_t *dtp, FILE *fp, if (buf != NULL) { if (first_timestamp == 0) first_timestamp = buf->dtbd_timestamp; - assert(buf->dtbd_timestamp >= first_timestamp); + if (buf->dtbd_timestamp < first_timestamp) { + warnx("cpu clocks out of sync " + "(%ju < %ju; offset: %ju). " + "Results may be incorrect!", + buf->dtbd_timestamp, first_timestamp, + first_timestamp - buf->dtbd_timestamp); + } dt_pq_insert(dtp->dt_bufq, buf); drops[i] = buf->dtbd_drops; -- 2.11.0 From b3c23c9eaafc31cc5f882bbc9bd98fb6584b6289 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 19:06:37 +0200 Subject: [PATCH 130/257] release: Add ElectroBSD install instructions ... that aren't particularly verbose, though. Obtained from: ElectroBSD --- release/rc.local | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/release/rc.local b/release/rc.local index cb442273c303..0d2fad20c1e1 100755 --- a/release/rc.local +++ b/release/rc.local @@ -37,7 +37,7 @@ if [ $? -eq 0 ]; then else # Serial or other console echo - echo "Welcome to FreeBSD!" + echo "Welcome to ElectroBSD!" echo echo "Please choose the appropriate terminal type for your system." echo "Common console types are:" @@ -62,7 +62,19 @@ if [ -f /etc/installerconfig ]; then exit fi -dialog --backtitle "FreeBSD Installer" --title "Welcome" --extra-button --extra-label "Shell" --ok-label "Install" --cancel-label "Live CD" --yesno "Welcome to FreeBSD! Would you like to begin an installation or use the live CD?" 0 0 +local timeout=15 +if dialog --backtitle "ElectroBSD $(uname -m) on a stick in da house" \ + --title "Careful now, your data may be at risk." \ + --timeout ${timeout} \ + --yesno "To install ElectroBSD, execute cloudiatr after creating a cloudiatr.conf. Understood? You have ${timeout} seconds to respond." \ + 0 0; then + echo "Great. Good luck." +else + echo "That's very unfortunate. If you don't have backups you probably should not continue." +fi + +exit 0; + case $? in $DIALOG_OK) # Install -- 2.11.0 From 0574db0a5e1c16ad4598e1e83867c772ec88808e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 28 Jul 2015 15:07:52 +0200 Subject: [PATCH 131/257] etc/rc.d/dumpon: Get dumpdev=AUTO working for GEOM_ELI consumers Obtained from: ElectroBSD --- etc/rc.d/dumpon | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/rc.d/dumpon b/etc/rc.d/dumpon index 66276eb50535..ce630522e12b 100755 --- a/etc/rc.d/dumpon +++ b/etc/rc.d/dumpon @@ -41,6 +41,7 @@ dumpon_start() fi while read dev mp type more ; do [ "${type}" = "swap" ] || continue + dev="${dev%%.eli}" [ -c "${dev}" ] || continue dumpon_try "${dev}" 2>/dev/null && return 0 done Date: Tue, 16 Jun 2015 15:07:06 +0200 Subject: [PATCH 132/257] deadlkres(): (Try to) optionally unlock deadlocked processes to work around USB deadlocks This is work in progress and hasn't been properly tested yet. Obtained from: ElectroBSD --- sys/kern/kern_clock.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index 7734e81518af..7ab290ca4863 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -178,6 +178,7 @@ static const char *blessed[] = { }; static int slptime_threshold = 1800; static int blktime_threshold = 900; +static int unlock_deadlocked_processes = 0; static int sleepfreq = 3; static void @@ -236,9 +237,15 @@ deadlkres(void) * turnstile. */ PROC_UNLOCK(p); - sx_sunlock(&allproc_lock); + if (unlock_deadlocked_processes == 0) { + sx_sunlock(&allproc_lock); panic("%s: possible deadlock detected for %p, blocked for %d ticks\n", - __func__, td, tticks); + __func__, td, tticks); + } else { + printf("%s: possible deadlock detected for %p, blocked for %d ticks. " + "Unlocking process to see what happens. Good luck.\n", __func__, td, tticks); + TD_CLR_LOCK(td); + } } } else if (TD_IS_SLEEPING(td) && TD_ON_SLEEPQ(td)) { @@ -282,9 +289,15 @@ deadlkres(void) continue; } PROC_UNLOCK(p); - sx_sunlock(&allproc_lock); + if (unlock_deadlocked_processes == 0) { + sx_sunlock(&allproc_lock); panic("%s: possible deadlock detected for %p, blocked for %d ticks\n", - __func__, td, tticks); + __func__, td, tticks); + } else { + printf("%s: possible deadlock detected for %p, blocked for %d ticks. " + "Unlocking process to see what happens. Good luck.\n", __func__, td, tticks); + TD_CLR_LOCK(td); + } } } else thread_unlock(td); @@ -316,6 +329,11 @@ SYSCTL_INT(_debug_deadlkres, OID_AUTO, blktime_threshold, CTLFLAG_RW, "Number of seconds within is valid to block on a turnstile"); SYSCTL_INT(_debug_deadlkres, OID_AUTO, sleepfreq, CTLFLAG_RW, &sleepfreq, 0, "Number of seconds between any deadlock resolver thread run"); +SYSCTL_INT(_debug_deadlkres, OID_AUTO, + unlock_deadlocked_processes_and_see_what_happens, + CTLFLAG_RW, &unlock_deadlocked_processes, 0, + "'Resolve' deadlocks by merely unlocking the locked process. " + "May cause permanent data corruption."); #endif /* DEADLKRES */ void -- 2.11.0 From d65b19bc4a65a5bd091b51537e071f7a0130ded9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 1 Aug 2015 11:42:55 +0200 Subject: [PATCH 133/257] sys/dev/vt: Default to showing splash CPU logo(s) on boot Obtained from: ElectroBSD --- sys/dev/vt/vt_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/vt/vt_core.c b/sys/dev/vt/vt_core.c index e09e8a786b3c..9f54b23d3097 100644 --- a/sys/dev/vt/vt_core.c +++ b/sys/dev/vt/vt_core.c @@ -137,7 +137,7 @@ static VT_SYSCTL_INT(kbd_panic, 0, "Enable request to panic. " /* Used internally, not a tunable. */ int vt_draw_logo_cpus; -VT_SYSCTL_INT(splash_cpu, 0, "Show logo CPUs during boot"); +VT_SYSCTL_INT(splash_cpu, 1, "Show logo CPUs during boot"); VT_SYSCTL_INT(splash_ncpu, 0, "Override number of logos displayed " "(0 = do not override)"); VT_SYSCTL_INT(splash_cpu_style, 2, "Draw logo style " -- 2.11.0 From 1046f5be5b1f5341653c2022a3a3aa642884210c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 1 Aug 2015 16:59:55 +0200 Subject: [PATCH 134/257] sys/dev/vt: Default to showing Beastie instead of the orb/sextoy Obtained from: ElectroBSD --- sys/dev/vt/vt_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/vt/vt_core.c b/sys/dev/vt/vt_core.c index 9f54b23d3097..4f7c5ee7c554 100644 --- a/sys/dev/vt/vt_core.c +++ b/sys/dev/vt/vt_core.c @@ -140,7 +140,7 @@ int vt_draw_logo_cpus; VT_SYSCTL_INT(splash_cpu, 1, "Show logo CPUs during boot"); VT_SYSCTL_INT(splash_ncpu, 0, "Override number of logos displayed " "(0 = do not override)"); -VT_SYSCTL_INT(splash_cpu_style, 2, "Draw logo style " +VT_SYSCTL_INT(splash_cpu_style, 1, "Draw logo style " "(0 = Alternate beastie, 1 = Beastie, 2 = Orb)"); VT_SYSCTL_INT(splash_cpu_duration, 10, "Hide logos after (seconds)"); -- 2.11.0 From 975169fe7cb54190be6f69c4143dbc5e976808ab Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 2 Aug 2015 17:56:01 +0200 Subject: [PATCH 135/257] sys/modules: Detach a couple of ATM-related modules Obtained from: ElectroBSD --- sys/modules/Makefile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 0011024c1d5f..0795a902a43e 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -118,7 +118,6 @@ SUBDIR= \ ${_ex} \ ${_exca} \ ext2fs \ - ${_fatm} \ fdc \ fdescfs \ ${_fe} \ @@ -131,8 +130,7 @@ SUBDIR= \ geom \ ${_glxiic} \ ${_glxsb} \ - gpio \ - hatm \ + ${_gpio} \ hifn \ hme \ ${_hpt27xx} \ @@ -288,7 +286,6 @@ SUBDIR= \ ow \ ${_padlock} \ ${_padlock_rng} \ - patm \ ${_pccard} \ ${_pcfclock} \ pcn \ @@ -496,7 +493,6 @@ _pfsync= pfsync .if ${MK_SOURCELESS_UCODE} != "no" _bce= bce -_fatm= fatm _fxp= fxp _ispfw= ispfw _mwlfw= mwlfw -- 2.11.0 From b7ec56814f074f39249b68247ef2f3d0ced1ada0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 2 Aug 2015 19:41:56 +0200 Subject: [PATCH 136/257] crypto/openssl: Update OPENSSL_VERSION_TEXT and break 'FIPS mode' Obtained from: ElectroBSD --- crypto/openssl/crypto/opensslv.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/openssl/crypto/opensslv.h b/crypto/openssl/crypto/opensslv.h index 7f52eae087c3..3b8d1108ab1b 100644 --- a/crypto/openssl/crypto/opensslv.h +++ b/crypto/openssl/crypto/opensslv.h @@ -32,9 +32,9 @@ extern "C" { */ # define OPENSSL_VERSION_NUMBER 0x100020afL # ifdef OPENSSL_FIPS -# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2j-fips 26 Sep 2016" +# error "Due to being silly, FIPS validation is not supported by ElectroBSD" # else -# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2j-freebsd 26 Sep 2016" +# define OPENSSL_VERSION_TEXT "OpenSSL 1.0.2j-ElectroBSD 2016-09-26" # endif # define OPENSSL_VERSION_PTEXT " part of " OPENSSL_VERSION_TEXT -- 2.11.0 From 550acad584dc29c81e943d6efad93d07ae88cc8f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 13:13:09 +0200 Subject: [PATCH 137/257] libstand: Show a message instead of the boring twiddle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Obviously the message should be sysctl controlled, but for now it's just a hardcoded 'Die Ärzte' reference which may contain traces of electro beer. Obtained from: ElectroBSD --- lib/libstand/twiddle.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/libstand/twiddle.c b/lib/libstand/twiddle.c index 96ebbbec43c0..b8c1117ea217 100644 --- a/lib/libstand/twiddle.c +++ b/lib/libstand/twiddle.c @@ -48,6 +48,7 @@ void twiddle(u_int callerdiv) { static u_int callercnt, globalcnt, pos; + static const char message[] = "5,6,7,8 - Bullenstaat!"; callercnt++; if (callerdiv > 1 && (callercnt % callerdiv) != 0) @@ -57,8 +58,9 @@ twiddle(u_int callerdiv) if (globaldiv > 1 && (globalcnt % globaldiv) != 0) return; - putchar("|/-\\"[pos++ & 3]); - putchar('\b'); + putchar((pos < sizeof(message)) ? message[pos] : '\b'); + pos++; + pos = pos % sizeof(message); } void -- 2.11.0 From 970c7ad609625f7930c2cf9d1d30eede53096817 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 4 Aug 2015 12:35:43 +0200 Subject: [PATCH 138/257] libstand: Change twiddle message to Elektrobier Obtained from: ElectroBSD --- lib/libstand/twiddle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/libstand/twiddle.c b/lib/libstand/twiddle.c index b8c1117ea217..f23449778cef 100644 --- a/lib/libstand/twiddle.c +++ b/lib/libstand/twiddle.c @@ -48,7 +48,7 @@ void twiddle(u_int callerdiv) { static u_int callercnt, globalcnt, pos; - static const char message[] = "5,6,7,8 - Bullenstaat!"; + static const char message[] = "Elektrobier!\n"; callercnt++; if (callerdiv > 1 && (callercnt % callerdiv) != 0) -- 2.11.0 From ca41bb92ce36751cbf8f58a7aa95f14cc937225c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 15 Aug 2015 11:29:39 +0200 Subject: [PATCH 139/257] ttys: Reduce default number of active terminals and mark the console as insecure Obtained from: ElectroBSD --- etc/etc.amd64/ttys | 16 ++++++++-------- etc/etc.i386/ttys | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/etc/etc.amd64/ttys b/etc/etc.amd64/ttys index 15eb30dc91ec..457b3d105b10 100644 --- a/etc/etc.amd64/ttys +++ b/etc/etc.amd64/ttys @@ -27,17 +27,17 @@ # # If console is marked "insecure", then init will ask for the root password # when going to single-user mode. -console none unknown off secure +console none unknown off insecure # ttyv0 "/usr/libexec/getty Pc" xterm on secure # Virtual terminals -ttyv1 "/usr/libexec/getty Pc" xterm on secure -ttyv2 "/usr/libexec/getty Pc" xterm on secure -ttyv3 "/usr/libexec/getty Pc" xterm on secure -ttyv4 "/usr/libexec/getty Pc" xterm on secure -ttyv5 "/usr/libexec/getty Pc" xterm on secure -ttyv6 "/usr/libexec/getty Pc" xterm on secure -ttyv7 "/usr/libexec/getty Pc" xterm on secure +ttyv1 "/usr/libexec/getty Pc" xterm off secure +ttyv2 "/usr/libexec/getty Pc" xterm off secure +ttyv3 "/usr/libexec/getty Pc" xterm off secure +ttyv4 "/usr/libexec/getty Pc" xterm off secure +ttyv5 "/usr/libexec/getty Pc" xterm off secure +ttyv6 "/usr/libexec/getty Pc" xterm off secure +ttyv7 "/usr/libexec/getty Pc" xterm off secure ttyv8 "/usr/local/bin/xdm -nodaemon" xterm off secure # Serial terminals # The 'dialup' keyword identifies dialin lines to login, fingerd etc. diff --git a/etc/etc.i386/ttys b/etc/etc.i386/ttys index 15eb30dc91ec..457b3d105b10 100644 --- a/etc/etc.i386/ttys +++ b/etc/etc.i386/ttys @@ -27,17 +27,17 @@ # # If console is marked "insecure", then init will ask for the root password # when going to single-user mode. -console none unknown off secure +console none unknown off insecure # ttyv0 "/usr/libexec/getty Pc" xterm on secure # Virtual terminals -ttyv1 "/usr/libexec/getty Pc" xterm on secure -ttyv2 "/usr/libexec/getty Pc" xterm on secure -ttyv3 "/usr/libexec/getty Pc" xterm on secure -ttyv4 "/usr/libexec/getty Pc" xterm on secure -ttyv5 "/usr/libexec/getty Pc" xterm on secure -ttyv6 "/usr/libexec/getty Pc" xterm on secure -ttyv7 "/usr/libexec/getty Pc" xterm on secure +ttyv1 "/usr/libexec/getty Pc" xterm off secure +ttyv2 "/usr/libexec/getty Pc" xterm off secure +ttyv3 "/usr/libexec/getty Pc" xterm off secure +ttyv4 "/usr/libexec/getty Pc" xterm off secure +ttyv5 "/usr/libexec/getty Pc" xterm off secure +ttyv6 "/usr/libexec/getty Pc" xterm off secure +ttyv7 "/usr/libexec/getty Pc" xterm off secure ttyv8 "/usr/local/bin/xdm -nodaemon" xterm off secure # Serial terminals # The 'dialup' keyword identifies dialin lines to login, fingerd etc. -- 2.11.0 From 3eabdecd65afc8637207477e6b7358ca04c2b25f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 21:28:15 +0200 Subject: [PATCH 140/257] usr.sbin: Don't build uathload if MK_SOURCELESS_UCODE is defined It breaks the build if the source tree does not contain the firmware. Obtained from: ElectroBSD --- usr.sbin/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile index 67499a10b5c6..f0219a6fa445 100644 --- a/usr.sbin/Makefile +++ b/usr.sbin/Makefile @@ -200,7 +200,12 @@ SUBDIR.${MK_TIMED}+= timed SUBDIR.${MK_TOOLCHAIN}+= config SUBDIR.${MK_TOOLCHAIN}+= crunch SUBDIR.${MK_UNBOUND}+= unbound +# MK_SOURCELESS_UCODE is supposed to affect kernel +# modules only but there is no matching define for +# userland stuff. +.if ${MK_SOURCELESS_UCODE} != "no" SUBDIR.${MK_USB}+= uathload +.endif SUBDIR.${MK_USB}+= uhsoctl SUBDIR.${MK_USB}+= usbconfig SUBDIR.${MK_USB}+= usbdump -- 2.11.0 From c7b61c3dc05cc6d6f216664eefb26cdd75487e02 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 21:33:19 +0200 Subject: [PATCH 141/257] Detach fwcontrol from the build It's pointless without firewire(4). Obtained from: ElectroBSD --- usr.sbin/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile index f0219a6fa445..3d146d6e7704 100644 --- a/usr.sbin/Makefile +++ b/usr.sbin/Makefile @@ -30,7 +30,6 @@ SUBDIR= adduser \ extattrctl \ fifolog \ fstyp \ - fwcontrol \ getfmac \ getpmac \ gstat \ -- 2.11.0 From c9a9b843b9ccf1fdd2c3a98872a3369a610023dd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 23:58:46 +0200 Subject: [PATCH 142/257] sys/x86: Remove another FreeBSD reference from log messages Obtained from: ElectroBSD --- sys/x86/x86/mp_x86.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c index 9de533922e95..1d2bed8ac170 100644 --- a/sys/x86/x86/mp_x86.c +++ b/sys/x86/x86/mp_x86.c @@ -617,7 +617,7 @@ cpu_mp_announce(void) int cores_per_pkg; int thrs_per_core; - printf("FreeBSD/SMP: "); + printf("ElectroBSD/SMP: "); if (topo_analyze(&topo_root, 1, &pkg_count, &cores_per_pkg, &thrs_per_core)) { printf("%d package(s)", pkg_count); @@ -631,7 +631,7 @@ cpu_mp_announce(void) printf("\n"); if (disabled_cpus) { - printf("FreeBSD/SMP Online: "); + printf("ElectroBSD/SMP Online: "); if (topo_analyze(&topo_root, 0, &pkg_count, &cores_per_pkg, &thrs_per_core)) { printf("%d package(s)", pkg_count); -- 2.11.0 From b1fc14b1ef4a731a2158189b186d73652e5180f4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 20 Aug 2015 10:12:27 +0200 Subject: [PATCH 143/257] pw(8): Increase minimal random password length to 32 Obtained from: ElectroBSD --- usr.sbin/pw/pw_user.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr.sbin/pw/pw_user.c b/usr.sbin/pw/pw_user.c index 75e7fb6e5c88..85db873017bd 100644 --- a/usr.sbin/pw/pw_user.c +++ b/usr.sbin/pw/pw_user.c @@ -512,7 +512,7 @@ pw_password(struct userconf * cnf, char const * user, bool dryrun) switch (cnf->default_password) { case -1: /* Random password */ - l = (arc4random() % 8 + 8); /* 8 - 16 chars */ + l = 32 + (arc4random() % 8); /* 32 - 40 chars */ for (i = 0; i < l; i++) pwbuf[i] = chars[arc4random_uniform(sizeof(chars)-1)]; pwbuf[i] = '\0'; -- 2.11.0 From 490921c3343f221a131fb27b71c32aa0ad343cd0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 24 Aug 2015 14:31:05 +0200 Subject: [PATCH 144/257] Makefile.inc1: Respect WITHOUT_TCSH Obtained from: ElectroBSD --- Makefile.inc1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index df8c415be268..ba910a0cae49 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1759,8 +1759,13 @@ _gcc_tools= gnu/usr.bin/cc/cc_tools _rescue=rescue/rescue .endif +.if ${MK_TCSH} != "no" +_tcsh=bin/csh +.endif + +build-tools: .MAKE .for _tool in \ - bin/csh \ + ${_tcsh} \ bin/sh \ ${LOCAL_TOOL_DIRS} \ lib/ncurses/ncurses \ @@ -1911,7 +1916,7 @@ native-xtools: .PHONY bin/cat \ bin/chmod \ bin/cp \ - bin/csh \ + ${_tcsh} \ bin/echo \ bin/expr \ bin/hostname \ -- 2.11.0 From 8476492da2f2b82276fabd76b59eee53e8d724f4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 4 Jun 2016 11:38:43 +0200 Subject: [PATCH 145/257] Detach a couple of rc scripts that aren't relevant for ElectroBSD XXX: Among other things this detaches bgfsck which could still be considered relevant for the memstick images. Obtained from: ElectroBSD --- etc/rc.d/Makefile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index ec3ad74a4bae..ac3dda4100a2 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -16,7 +16,6 @@ FILES= DAEMON \ archdep \ auditd \ auditdistd \ - bgfsck \ ${_blacklistd} \ ${_bluetooth} \ bridge \ @@ -32,7 +31,6 @@ FILES= DAEMON \ dhclient \ dmesg \ dumpon \ - fsck \ gbde \ geli \ geli2 \ @@ -45,7 +43,6 @@ FILES= DAEMON \ hostname \ iovctl \ ip6addrctl \ - ipfilter \ ipfs \ ipfw \ ipmon \ @@ -70,7 +67,6 @@ FILES= DAEMON \ mdconfig2 \ mountd \ msgs \ - natd \ netif \ netoptions \ netwait \ @@ -87,8 +83,6 @@ FILES= DAEMON \ pf \ pflog \ pfsync \ - ppp \ - pppoed \ pwcheck \ quota \ random \ @@ -105,7 +99,6 @@ FILES= DAEMON \ securelevel \ serial \ soft-protection \ - sppp \ statd \ static_arp \ static_ndp \ -- 2.11.0 From 7cbb6f7df6ca79ea5b4778554190f38ef6c42004 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 9 Jul 2015 14:55:50 +0200 Subject: [PATCH 146/257] etc/pkg: Add and install ElectroBSD.conf and a pubkey file ... for testing purposes. While at it, detach FreeBSD.conf and the corresponding fingerprint file from the build. Obtained from: ElectroBSD --- etc/pkg/ElectroBSD.conf | 21 +++++++++++++++++++++ etc/pkg/Makefile | 2 +- etc/pkg/electrobsd-test-repo.pub | 14 ++++++++++++++ share/keys/pkg/trusted/Makefile | 2 +- share/keys/pkg/trusted/electrobsd-test-key.conf | 2 ++ 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 etc/pkg/ElectroBSD.conf create mode 100644 etc/pkg/electrobsd-test-repo.pub create mode 100644 share/keys/pkg/trusted/electrobsd-test-key.conf diff --git a/etc/pkg/ElectroBSD.conf b/etc/pkg/ElectroBSD.conf new file mode 100644 index 000000000000..4bf82d992bee --- /dev/null +++ b/etc/pkg/ElectroBSD.conf @@ -0,0 +1,21 @@ +ElectroBSD: { + # Note that accessing this repository requires a local proxy + # that port forwards the incomming http requests to an authorized + # Tor client that forwards them to: http://gkpssb7hd77qznoa.onion/ + # + # To get the required secret you have to ask someone who knows it and + # come up with a good reason why you want to run unreproducible packages + # instead of building your packages from source. + # + # Example bootstrapping setup: + # Your server: pkg -> sshd + # | + # Your client: --> ssh -> privoxy -> tor + # | + # Outside your control: --> Tor network -> Tor HS + url: "http://127.0.0.1:8000/packages/${ABI}/", + mirror_type: "NONE", + signature_type: "pubkey", + pubkey: "/etc/pkg/electrobsd-test-repo.pub", + enabled: yes +} diff --git a/etc/pkg/Makefile b/etc/pkg/Makefile index abc8c64ca5cb..614989f36674 100644 --- a/etc/pkg/Makefile +++ b/etc/pkg/Makefile @@ -2,7 +2,7 @@ NO_OBJ= -FILES= FreeBSD.conf +FILES= ElectroBSD.conf electrobsd-test-repo.pub FILESDIR= /etc/pkg FILESMODE= 644 diff --git a/etc/pkg/electrobsd-test-repo.pub b/etc/pkg/electrobsd-test-repo.pub new file mode 100644 index 000000000000..ee213cb062b1 --- /dev/null +++ b/etc/pkg/electrobsd-test-repo.pub @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtwIgcSgQT2QJvxwDiKhh +xYidaxWaJL/EHXnYH8TpiujNeg8yZUCiT07MChw/wO3KrVY3SlhFkl8MHnwaFWNS +jSJFNQIHoVo8of0juDXhIMMY0DTVFVpSfgq+Ea0SrVZawQchcc7XODjp7mQOH0Nz +Z0FUpQ4z7TsdrPV/Kj/SQteK7Q0qAnDGT5GBxb+4m5qmNbv74GVpWmxJ5kC4OY3g +v8oTRZ5wG/mf5nKG+QXGKbcfoK+fd+NL9gp5awhvnp5cRo2+ZrQvnYKy5ym9O+jM +9hv5bL7OYIW7gErPdQd0+SFejcDg4TamY824krkjgIf7pnRVAuuSHiy8PaHmmNMd +StFo8IW6687C0X4gNuaPtR1TCkVZugrgev/w5GPp765HmE4WvrtPU4u9/EvWO2L6 +2bHmsJqNIGyq1lfHk/cx49CEhDPjRcYin8MKKzRFUR7CWiNDBz2GK6Lcfd78cG5I +HGJAthhi3rOOnLsTMuIQ/+GkVCDNwweLxSoLQUzGnheHqBjYyZSfZ8u9I/OKtwp3 +OMhGOD1/yuRaat+QER5VwiDP3AxcIlq6aNNytJW6l/kZZLVsr99YmROyYsTps0Zm +OXqxQFsZ6zzlYO95HP4ITJ7ubk84YPTvxbQ6LqsijqRI322aNvxxeEWyFxXpyCpp +uq50HtVoVX1a4ONsh/9l75sCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/share/keys/pkg/trusted/Makefile b/share/keys/pkg/trusted/Makefile index e55977c0a9a9..f55afc979969 100644 --- a/share/keys/pkg/trusted/Makefile +++ b/share/keys/pkg/trusted/Makefile @@ -1,6 +1,6 @@ # $FreeBSD$ -FILES= pkg.freebsd.org.2013102301 +FILES= electrobsd-test-key.conf FILESDIR= ${SHAREDIR}/keys/pkg/trusted FILESMODE= 644 diff --git a/share/keys/pkg/trusted/electrobsd-test-key.conf b/share/keys/pkg/trusted/electrobsd-test-key.conf new file mode 100644 index 000000000000..e81a919e0ab2 --- /dev/null +++ b/share/keys/pkg/trusted/electrobsd-test-key.conf @@ -0,0 +1,2 @@ +function: "sha256" +fingerprint: "8b1ef90f16ccc99342fb204f7ff57cdd31449e5ce1baa10c0b1b09593c44219d" -- 2.11.0 From f5212a5b7abc92ff3dd9a8ca02a211d7cd99e747 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 5 Sep 2015 22:46:09 +0200 Subject: [PATCH 147/257] dtrace: Add look-who-is-reaping It can be used to monitor and finetune the ARC cache reaper. The stack trace obviously isn't particular useful anymore now that the reaper is called from a single location. Obtained from: ElectroBSD --- share/dtrace/look-who-is-reaping | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 share/dtrace/look-who-is-reaping diff --git a/share/dtrace/look-who-is-reaping b/share/dtrace/look-who-is-reaping new file mode 100755 index 000000000000..0db6324a64f7 --- /dev/null +++ b/share/dtrace/look-who-is-reaping @@ -0,0 +1,58 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * look-who-is-reaping + * + * Collect stack traces for reap_arc_caches(). + * + * Copyright (c) 2015 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + printf("%Y: Look who's (ARC cache) reaping .... Hit CTRL-C to exit.\n", walltimestamp); +} + +fbt::reap_arc_caches:entry +{ + @reapers[probefunc,stack(3)] = count(); + @reapers_total[probefunc,stack(3)] = count(); + reaped++ +} + +tick-10s +/reaped == 0/ +{ + printf("%Y: The ARC cache reaper has not been seen for 10 seconds!\n", walltimestamp); +} + +tick-10s +/reaped/ +{ + printf("%Y: The ARC reaper was called:\n", walltimestamp); + printa(@reapers); + trunc(@reapers); + reaped = 0 +} + +END +/reaped/ +{ + printf("%Y: The ARC reaper stats total:\n", walltimestamp); + printa(@reapers_total); +} -- 2.11.0 From ff112456863d40b0a1e029ffd6fe462d874b10ea Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Sep 2015 20:07:22 +0200 Subject: [PATCH 148/257] dtrace: Import fbt-time which llquantizes the time spent in a given kernel function Among other things this can be useful for general system tuning and analysis of lock contention (once you know where to expect it). Obtained from: ElectroBSD --- share/dtrace/fbt-time | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 share/dtrace/fbt-time diff --git a/share/dtrace/fbt-time b/share/dtrace/fbt-time new file mode 100755 index 000000000000..27a950b63a0c --- /dev/null +++ b/share/dtrace/fbt-time @@ -0,0 +1,80 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * fbt-time + * + * Measures the time spent in a given kernel function and llquantizes + * the results for the traced calls. Example output can be seen at: + * http://www.fabiankeil.de/gehacktes/electrobsd/zfs-arc-tuning/ + * + * WARNING: Tracing kernel functions that get executed several thousand + * times per second may negatively impact system performance + * and can even cause brief lockups. Before using this script + * on production systems you may want to sample the traced + * kernel function first. + * + * XXX: This script should be fleshed out to be more resilient to + * incorrect input and provide more awesome stats. A better name + * would be nice, too. + * + * Copyright (c) 2014 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + traced_function = $$1; + + multiple = "microseconds"; + divisor = 1000; + executions = 0; + + this->timestamp = walltimestamp; + this->msecs = (this->timestamp / 1000000) % 1000; + printf("%Y.%.3d: Tracing '%s'. Hit CTRL-C to exit.\n", + this->timestamp, this->msecs, traced_function); +} + +fbt::$$1:entry +{ + self->timestamp = timestamp; + executions++; +} + +fbt::$$1:return +/self->timestamp/ +{ + this->elapsed = (timestamp - self->timestamp) / divisor; + @elapsed_time = llquantize(this->elapsed, 10, 0, 5, 10); + @elapsed_avg["Average"] = avg(this->elapsed); + self->elapsed = 0; +} + +tick-10sec, +END +/executions/ +{ + printf("%Y: Runtime for %s() in %s\n", walltimestamp, traced_function, multiple); + printa(@elapsed_avg); + printa(@elapsed_time); +} + +END +/executions == 0/ +{ + printf("%Y: Looks like %s() wasn't called yet.\n", walltimestamp, traced_function); +} -- 2.11.0 From c63a4e673d0c596735cb238f06bd33fb0f62747d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 4 Mar 2015 13:06:05 +0100 Subject: [PATCH 149/257] ifconfig: Implement MAC address randomization Usage: ifconfig $nic ether random Inspired by the OpenBSD code which unfortunately can't be imported directly due to code differences. Obtained from: ElectroBSD --- sbin/ifconfig/af_link.c | 8 ++++++++ sbin/ifconfig/ifconfig.8 | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sbin/ifconfig/af_link.c b/sbin/ifconfig/af_link.c index c96cbd2ce92e..c32319b30398 100644 --- a/sbin/ifconfig/af_link.c +++ b/sbin/ifconfig/af_link.c @@ -90,6 +90,14 @@ link_getaddr(const char *addr, int which) if (which != ADDR) errx(1, "can't set link-level netmask or broadcast"); + if (!strcmp(addr, "random")) { + sa->sa_family = AF_LINK; + sa->sa_len = ETHER_ADDR_LEN; + arc4random_buf(&sa->sa_data, sa->sa_len); + /* Make sure it's a non-multicast hardware address */ + sa->sa_data[0] &= 0xfc; + return; + } if ((temp = malloc(strlen(addr) + 2)) == NULL) errx(1, "malloc failed"); temp[0] = ':'; diff --git a/sbin/ifconfig/ifconfig.8 b/sbin/ifconfig/ifconfig.8 index e8db561f45a0..05d23d358b01 100644 --- a/sbin/ifconfig/ifconfig.8 +++ b/sbin/ifconfig/ifconfig.8 @@ -141,7 +141,10 @@ parameter below for more information. The link-level .Pq Dq link address -is specified as a series of colon-separated hex digits. +is specified as a series of colon-separated hex digits +or, if the address is +.Dq random , +will be chosen randomly. This can be used to, for example, set a new MAC address on an Ethernet interface, though the mechanism used is not Ethernet specific. -- 2.11.0 From 22b6859dc5c26c438510636bdf3b9b85fea9eabf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 4 Mar 2015 14:22:43 +0100 Subject: [PATCH 150/257] Import rc.d/rether which randomizes MAC addresses ... provided ifconfig supports "either random". Obtained from: ElectroBSD --- etc/rc.d/Makefile | 1 + etc/rc.d/rether | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100755 etc/rc.d/rether diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index ac3dda4100a2..578e69ababd5 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -89,6 +89,7 @@ FILES= DAEMON \ rarpd \ rctl \ resolv \ + rether \ root \ route6d \ routing \ diff --git a/etc/rc.d/rether b/etc/rc.d/rether new file mode 100755 index 000000000000..51e860353458 --- /dev/null +++ b/etc/rc.d/rether @@ -0,0 +1,97 @@ +#!/bin/sh +# +########################################################################### +# +# rether - Randomizes MAC addresses +# +# Add the following line to /etc/rc.conf to randomize the MAC +# address for all recognized network interfaces that got one +# at startup: +# +# rether_enable="YES" +# +# You can specify the interfaces manually like this: +# +# rether_interfaces="bge0 iwn0" +# +# By default rether runs before netif so cloned devices aren't +# expected to exist yet. In case of wlan clones the MAC address +# of the parent is inherited so as long as it's randomized the +# clone should be fine too. +# +# Note that MAC address mismatches between clone and parent may +# prevent the clone from working as expected. If you intend to +# run this script after the system is up, you may want to +# explicitly set rether_interfaces to skip clones. +# +# Rether requires an ifconfig version that understands "ether random". +# +########################################################################### +# +# Copyright (c) 2014 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR +# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT +# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. +########################################################################### + +# PROVIDE: rether +# REQUIRE: FILESYSTEMS +# BEFORE: netif + +. /etc/rc.subr + +name="rether" +rcvar="rether_enable" + +rether_enable="${rether_enable-NO}" + +start_cmd="rether_start" +stop_cmd=":" + +# For the reason given above, it may make sense to skip cloned devices. +get_ethernet_interfaces() { + ifconfig -l ether +} + +get_ethernet_address() { + local interface="${1}" + + ifconfig "${interface}" ether | awk '/ether/ {print $2}' +} + +rether_start() { + local \ + interface \ + ethernet_address + + if [ -z "${rether_interfaces}" ]; then + rether_interfaces="$(get_ethernet_interfaces)" + fi + + echo "Randomizing MAC addresses for: ${rether_interfaces}" + + for interface in $rether_interfaces; + do + ethernet_address="$(get_ethernet_address "${interface}")" + + ifconfig "${interface}" ether random + + if [ "${ethernet_address}" = "$(get_ethernet_address "${interface}")" ]; then + echo "Failed to randomize MAC address for ${interface}: ${ethernet_address}" + fi + done +} + +load_rc_config "${name}" +run_rc_command "${1}" -- 2.11.0 From 6b317420bad8b76570b264447979d24a55db6ecf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 17 Sep 2015 17:20:31 +0200 Subject: [PATCH 151/257] share/mk/src.opts.mk: Detach LIB32 from the build Obtained from: ElectroBSD --- share/mk/src.opts.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk index 494cdf976848..5a4e185913fa 100644 --- a/share/mk/src.opts.mk +++ b/share/mk/src.opts.mk @@ -104,7 +104,6 @@ __DEFAULT_YES_OPTIONS = \ LDNS \ LDNS_UTILS \ LEGACY_CONSOLE \ - LIB32 \ LIBPTHREAD \ LIBTHR \ LOCALES \ @@ -181,6 +180,7 @@ __DEFAULT_NO_OPTIONS += \ GCOV \ IPFILTER \ ISCSI \ + LIB32 \ NDIS \ RBOOTD \ PC_SYSINSTALL \ -- 2.11.0 From 99655b25dcc4c535689b8708bb155a1324044d38 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 15:19:51 +0200 Subject: [PATCH 152/257] Prevent inlining of vm_pageout_scan() to allow fbt tracing Obtained from: ElectroBSD --- sys/vm/vm_pageout.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index ed9d439af87f..a3f8774c9160 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -849,7 +849,7 @@ unlock_mp: * Returns true if pass was zero or enough pages were freed by the inactive * queue scan to meet the target. */ -static bool +static bool __noinline vm_pageout_scan(struct vm_domain *vmd, int pass) { vm_page_t m, next; -- 2.11.0 From 44b37ddcb7e9aeaf2f11c0e18e19da9cfac100cd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 22 Sep 2015 15:33:12 +0200 Subject: [PATCH 153/257] Attempt to fix MAC address randomization after r287197 Obtained from: ElectroBSD --- etc/Makefile | 1 + etc/{rc.d/rether => mac-randomization.subr} | 35 ++++++----------------------- etc/rc.d/Makefile | 1 - etc/rc.d/netif | 3 +++ 4 files changed, 11 insertions(+), 29 deletions(-) rename etc/{rc.d/rether => mac-randomization.subr} (74%) diff --git a/etc/Makefile b/etc/Makefile index fe518d283d19..67bb39e701d9 100644 --- a/etc/Makefile +++ b/etc/Makefile @@ -31,6 +31,7 @@ BIN1= crontab \ login.access \ login.conf \ mac.conf \ + mac-randomization.subr \ motd \ netconfig \ network.subr \ diff --git a/etc/rc.d/rether b/etc/mac-randomization.subr similarity index 74% rename from etc/rc.d/rether rename to etc/mac-randomization.subr index 51e860353458..d486bf454591 100755 --- a/etc/rc.d/rether +++ b/etc/mac-randomization.subr @@ -2,7 +2,7 @@ # ########################################################################### # -# rether - Randomizes MAC addresses +# Sub routines to randomizes MAC addresses after r287197 (WIP!) # # Add the following line to /etc/rc.conf to randomize the MAC # address for all recognized network interfaces that got one @@ -14,16 +14,6 @@ # # rether_interfaces="bge0 iwn0" # -# By default rether runs before netif so cloned devices aren't -# expected to exist yet. In case of wlan clones the MAC address -# of the parent is inherited so as long as it's randomized the -# clone should be fine too. -# -# Note that MAC address mismatches between clone and parent may -# prevent the clone from working as expected. If you intend to -# run this script after the system is up, you may want to -# explicitly set rether_interfaces to skip clones. -# # Rether requires an ifconfig version that understands "ether random". # ########################################################################### @@ -45,21 +35,8 @@ # ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. ########################################################################### -# PROVIDE: rether -# REQUIRE: FILESYSTEMS -# BEFORE: netif - -. /etc/rc.subr - -name="rether" -rcvar="rether_enable" - rether_enable="${rether_enable-NO}" -start_cmd="rether_start" -stop_cmd=":" - -# For the reason given above, it may make sense to skip cloned devices. get_ethernet_interfaces() { ifconfig -l ether } @@ -70,11 +47,16 @@ get_ethernet_address() { ifconfig "${interface}" ether | awk '/ether/ {print $2}' } -rether_start() { +randomize_mac_addresses() { local \ interface \ ethernet_address + if [ "${rether_enable}" = "NO" ]; then + echo "Not randomizing any MAC addresses!" + return 0 + fi + if [ -z "${rether_interfaces}" ]; then rether_interfaces="$(get_ethernet_interfaces)" fi @@ -92,6 +74,3 @@ rether_start() { fi done } - -load_rc_config "${name}" -run_rc_command "${1}" diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index 578e69ababd5..ac3dda4100a2 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -89,7 +89,6 @@ FILES= DAEMON \ rarpd \ rctl \ resolv \ - rether \ root \ route6d \ routing \ diff --git a/etc/rc.d/netif b/etc/rc.d/netif index a1543e63e704..6b52c50e7f4f 100755 --- a/etc/rc.d/netif +++ b/etc/rc.d/netif @@ -32,6 +32,7 @@ . /etc/rc.subr . /etc/network.subr +. /etc/mac-randomization.subr name="netif" desc="Network interface setup" @@ -74,6 +75,8 @@ netif_start() # Create cloned interfaces clone_up $cmdifn + randomize_mac_addresses + # Rename interfaces. ifnet_rename $cmdifn -- 2.11.0 From 6c6e18fbbc57964f33043a05be1ad0a3d6c4cc06 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Sep 2015 09:53:58 +0200 Subject: [PATCH 154/257] sys/boot/i386: Detach libfirewire from the build Obtained from: ElectroBSD --- sys/boot/i386/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/i386/Makefile b/sys/boot/i386/Makefile index 0c3daed185ee..d42c7b9dc994 100644 --- a/sys/boot/i386/Makefile +++ b/sys/boot/i386/Makefile @@ -3,7 +3,7 @@ .include SUBDIR= mbr pmbr boot0 boot0sio btx boot2 cdboot gptboot \ - libi386 libfirewire loader + libi386 loader # special boot programs, 'self-extracting boot2+loader' SUBDIR+= pxeldr -- 2.11.0 From 8811e0ee4247a587662ed35589a2ff9245d3d165 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 29 Sep 2015 22:31:45 +0200 Subject: [PATCH 155/257] README: Spell out GPL correctly, the G does not stand for GNU Obtained from: ElectroBSD --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README b/README index b863557e4baa..fe02a3852c3f 100644 --- a/README +++ b/README @@ -45,8 +45,8 @@ crypto Cryptography stuff (see crypto/README). etc Template files for /etc. -gnu Various commands and libraries under the GNU Public License. - Please see gnu/COPYING* for more information. +gnu Various commands and libraries under the GNU General Public + License. Please see gnu/COPYING* for more information. include System include files. -- 2.11.0 From 1df5685206366574487ff80fe2b3b882f3eabea7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 30 Sep 2015 21:15:20 +0200 Subject: [PATCH 156/257] Add two DTrace scripts. WIP Obtained from: ElectroBSD --- share/dtrace/io-breakdown | 19 +++++++++++++++++++ share/dtrace/open-ios | 31 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100755 share/dtrace/io-breakdown create mode 100755 share/dtrace/open-ios diff --git a/share/dtrace/io-breakdown b/share/dtrace/io-breakdown new file mode 100755 index 000000000000..ee9e592f1bb5 --- /dev/null +++ b/share/dtrace/io-breakdown @@ -0,0 +1,19 @@ +#!/usr/sbin/dtrace -s + +/* This is the example from dtrace_io(4) */ + +#pragma D option quiet +#pragma D option dynvarsize=16m + +#pragma D option quiet + +io:::start +{ + @[args[1]->device_name, execname, pid] = sum(args[0]->bio_bcount); +} + +END +{ + printf("%10s %20s %10s %15s", "DEVICE", "APP", "PID", "BYTES"); + printa("%10s %20s %10d %15@d", @); +} diff --git a/share/dtrace/open-ios b/share/dtrace/open-ios new file mode 100755 index 000000000000..ee8bbe0d922d --- /dev/null +++ b/share/dtrace/open-ios @@ -0,0 +1,31 @@ +#!/usr/sbin/dtrace -s + +#pragma D option quiet +#pragma D option dynvarsize=16m + +dtrace:::BEGIN +{ + printf("Tracing... Hit Ctrl-C to end.\n"); +} + +io:::start +{ + printf("%Y: Device name\n", walltimestamp); + print(args[1]->device_name); + start_time[arg0] = timestamp; + @open_ios = count(); +} + +io:::done +/this->start = start_time[arg0]/ +{ + @done_ios = count(); +} + +tick-10s +{ + printf("%Y: Started ios", walltimestamp); + printa(@open_ios); + printf("%Y: Done ios", walltimestamp); + printa(@done_ios); +} -- 2.11.0 From 141dee7a7200dfb9cf77a6e4f69bf484fa6fed82 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 13:04:33 +0200 Subject: [PATCH 157/257] games/fortune/datfiles: Do not install non-free gerrold.limerick Obtained from: ElectroBSD --- usr.bin/fortune/datfiles/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr.bin/fortune/datfiles/Makefile b/usr.bin/fortune/datfiles/Makefile index 1eabaa4cdc61..5d979d9f21ac 100644 --- a/usr.bin/fortune/datfiles/Makefile +++ b/usr.bin/fortune/datfiles/Makefile @@ -5,7 +5,7 @@ DB= fortunes freebsd-tips murphy startrek zippy # TO AVOID INSTALLING THE POTENTIALLY OFFENSIVE FORTUNES, COMMENT OUT THE # NEXT LINE. -DB+= limerick murphy-o gerrold.limerick +DB+= limerick murphy-o BLDS= ${DB:S/$/.dat/} FILES= ${DB} ${BLDS} -- 2.11.0 From e36541591ae6336d8514f87c1e9da186188cc4da Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 12:53:54 +0200 Subject: [PATCH 158/257] fortune/datafiles: Add German fundamental rights (poorly formatted) Enforcing these rights is left as an excercise for the reader. Obtained from: ElectroBSD --- usr.bin/fortune/datfiles/Makefile | 2 +- usr.bin/fortune/datfiles/grundrechte | 346 +++++++++++++++++++++++++++++++++++ 2 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 usr.bin/fortune/datfiles/grundrechte diff --git a/usr.bin/fortune/datfiles/Makefile b/usr.bin/fortune/datfiles/Makefile index 5d979d9f21ac..0f25ce005278 100644 --- a/usr.bin/fortune/datfiles/Makefile +++ b/usr.bin/fortune/datfiles/Makefile @@ -1,7 +1,7 @@ # @(#)Makefile 8.2 (Berkeley) 4/19/94 # $FreeBSD$ -DB= fortunes freebsd-tips murphy startrek zippy +DB= grundrechte fortunes freebsd-tips murphy startrek zippy # TO AVOID INSTALLING THE POTENTIALLY OFFENSIVE FORTUNES, COMMENT OUT THE # NEXT LINE. diff --git a/usr.bin/fortune/datfiles/grundrechte b/usr.bin/fortune/datfiles/grundrechte new file mode 100644 index 000000000000..c9a098878b17 --- /dev/null +++ b/usr.bin/fortune/datfiles/grundrechte @@ -0,0 +1,346 @@ +% +Grundgesetz Artikel 1 +(1) Die Würde des Menschen ist unantastbar. Sie zu achten und zu +schützen ist Verpflichtung aller staatlichen Gewalt. +(2) Das Deutsche Volk bekennt sich darum zu unverletzlichen und +unveräußerlichen Menschenrechten als Grundlage jeder menschlichen +Gemeinschaft, des Friedens und der Gerechtigkeit in der Welt. +(3) Die nachfolgenden Grundrechte binden Gesetzgebung, vollziehende +Gewalt und Rechtsprechung als unmittelbar geltendes Recht. + % +Grundgesetz Artikel 2 +(1) Jeder hat das Recht auf die freie Entfaltung seiner +Persönlichkeit, soweit er nicht die Rechte anderer verletzt und nicht +gegen die verfassungsmäßige Ordnung oder das Sittengesetz verstößt. +(2) Jeder hat das Recht auf Leben und körperliche Unversehrtheit. Die +Freiheit der Person ist unverletzlich. In diese Rechte darf nur auf +Grund eines Gesetzes eingegriffen werden. +% +Grundgesetz Artikel 3 +(1) Alle Menschen sind vor dem Gesetz gleich. +(2) Männer und Frauen sind gleichberechtigt. Der Staat fördert die +tatsächliche Durchsetzung der Gleichberechtigung von Frauen und +Männern und wirkt auf die Beseitigung bestehender Nachteile hin. +(3) Niemand darf wegen seines Geschlechtes, seiner Abstammung, seiner +Rasse, seiner Sprache, seiner Heimat und Herkunft, seines Glaubens, +seiner religiösen oder politischen Anschauungen benachteiligt oder +bevorzugt werden. Niemand darf wegen seiner Behinderung benachteiligt +werden. +% +Grundgesetz Artikel 4 +(1) Die Freiheit des Glaubens, des Gewissens und die Freiheit des +religiösen und weltanschaulichen Bekenntnisses sind unverletzlich. +(2) Die ungestörte Religionsausübung wird gewährleistet. +(3) Niemand darf gegen sein Gewissen zum Kriegsdienst mit der Waffe +gezwungen werden. Das Nähere regelt ein Bundesgesetz. +% +Grundgesetz Artikel 5 +(1) Jeder hat das Recht, seine Meinung in Wort, Schrift und Bild frei +zu äußern und zu verbreiten und sich aus allgemein zugänglichen +Quellen ungehindert zu unterrichten. Die Pressefreiheit und die +Freiheit der Berichterstattung durch Rundfunk und Film werden +gewährleistet. Eine Zensur findet nicht statt. +(2) Diese Rechte finden ihre Schranken in den Vorschriften der +allgemeinen Gesetze, den gesetzlichen Bestimmungen zum Schutze der +Jugend und in dem Recht der persönlichen Ehre. +(3) Kunst und Wissenschaft, Forschung und Lehre sind frei. Die +Freiheit der Lehre entbindet nicht von der Treue zur Verfassung. +% +Grundgesetz Artikel 6 +(1) Ehe und Familie stehen unter dem besonderen Schutze der staatlichen Ordnung. +(2) Pflege und Erziehung der Kinder sind das natürliche Recht der +Eltern und die zuvörderst ihnen obliegende Pflicht. Über ihre +Betätigung wacht die staatliche Gemeinschaft. +(3) Gegen den Willen der Erziehungsberechtigten dürfen Kinder nur auf +Grund eines Gesetzes von der Familie getrennt werden, wenn die +Erziehungsberechtigten versagen oder wenn die Kinder aus anderen +Gründen zu verwahrlosen drohen. +(4) Jede Mutter hat Anspruch auf den Schutz und die Fürsorge der Gemeinschaft. +(5) Den unehelichen Kindern sind durch die Gesetzgebung die gleichen +Bedingungen für ihre leibliche und seelische Entwicklung und ihre +Stellung in der Gesellschaft zu schaffen wie den ehelichen Kindern. +% +Grundgesetz Artikel 7 +(1) Das gesamte Schulwesen steht unter der Aufsicht des Staates. +(2) Die Erziehungsberechtigten haben das Recht, über die Teilnahme des +Kindes am Religionsunterricht zu bestimmen. +(3) Der Religionsunterricht ist in den öffentlichen Schulen mit +Ausnahme der bekenntnisfreien Schulen ordentliches +Lehrfach. Unbeschadet des staatlichen Aufsichtsrechtes wird der +Religionsunterricht in Übereinstimmung mit den Grundsätzen der +Religionsgemeinschaften erteilt. Kein Lehrer darf gegen seinen Willen +verpflichtet werden, Religionsunterricht zu erteilen. +(4) Das Recht zur Errichtung von privaten Schulen wird +gewährleistet. Private Schulen als Ersatz für öffentliche Schulen +bedürfen der Genehmigung des Staates und unterstehen den +Landesgesetzen. Die Genehmigung ist zu erteilen, wenn die privaten +Schulen in ihren Lehrzielen und Einrichtungen sowie in der +wissenschaftlichen Ausbildung ihrer Lehrkräfte nicht hinter den +öffentlichen Schulen zurückstehen und eine Sonderung der Schüler nach +den Besitzverhältnissen der Eltern nicht gefördert wird. Die +Genehmigung ist zu versagen, wenn die wirtschaftliche und rechtliche +Stellung der Lehrkräfte nicht genügend gesichert ist. +(5) Eine private Volksschule ist nur zuzulassen, wenn die +Unterrichtsverwaltung ein besonderes pädagogisches Interesse anerkennt +oder, auf Antrag von Erziehungsberechtigten, wenn sie als +Gemeinschaftsschule, als Bekenntnis- oder Weltanschauungsschule +errichtet werden soll und eine öffentliche Volksschule dieser Art in +der Gemeinde nicht besteht. +(6) Vorschulen bleiben aufgehoben. +% +Grundgesetz Artikel 8 +(1) Alle Deutschen haben das Recht, sich ohne Anmeldung oder Erlaubnis +friedlich und ohne Waffen zu versammeln. +(2) Für Versammlungen unter freiem Himmel kann dieses Recht durch +Gesetz oder auf Grund eines Gesetzes beschränkt werden. +% +Grundgesetz Artikel 9 +(1) Alle Deutschen haben das Recht, Vereine und Gesellschaften zu bilden. +(2) Vereinigungen, deren Zwecke oder deren Tätigkeit den Strafgesetzen +zuwiderlaufen oder die sich gegen die verfassungsmäßige Ordnung oder +gegen den Gedanken der Völkerverständigung richten, sind verboten. +(3) Das Recht, zur Wahrung und Förderung der Arbeits- und +Wirtschaftsbedingungen Vereinigungen zu bilden, ist für jedermann und +für alle Berufe gewährleistet. Abreden, die dieses Recht einschränken +oder zu behindern suchen, sind nichtig, hierauf gerichtete Maßnahmen +sind rechtswidrig. Maßnahmen nach den Artikeln 12a, 35 Abs. 2 und 3, +Artikel 87a Abs. 4 und Artikel 91 dürfen sich nicht gegen +Arbeitskämpfe richten, die zur Wahrung und Förderung der Arbeits- und +Wirtschaftsbedingungen von Vereinigungen im Sinne des Satzes 1 geführt +werden. +% +Grundgesetz Artikel 10 +(1) Das Briefgeheimnis sowie das Post- und Fernmeldegeheimnis sind +unverletzlich. +(2) Beschränkungen dürfen nur auf Grund eines Gesetzes angeordnet +werden. Dient die Beschränkung dem Schutze der freiheitlichen +demokratischen Grundordnung oder des Bestandes oder der Sicherung des +Bundes oder eines Landes, so kann das Gesetz bestimmen, daß sie dem +Betroffenen nicht mitgeteilt wird und daß an die Stelle des +Rechtsweges die Nachprüfung durch von der Volksvertretung bestellte +Organe und Hilfsorgane tritt. +% +Grundgesetz Artikel 11 +(1) Alle Deutschen genießen Freizügigkeit im ganzen Bundesgebiet. +(2) Dieses Recht darf nur durch Gesetz oder auf Grund eines Gesetzes +und nur für die Fälle eingeschränkt werden, in denen eine ausreichende +Lebensgrundlage nicht vorhanden ist und der Allgemeinheit daraus +besondere Lasten entstehen würden oder in denen es zur Abwehr einer +drohenden Gefahr für den Bestand oder die freiheitliche demokratische +Grundordnung des Bundes oder eines Landes, zur Bekämpfung von +Seuchengefahr, Naturkatastrophen oder besonders schweren +Unglücksfällen, zum Schutze der Jugend vor Verwahrlosung oder um +strafbaren Handlungen vorzubeugen, erforderlich ist. +% +Grundgesetz Artikel 12 +(1) Alle Deutschen haben das Recht, Beruf, Arbeitsplatz und +Ausbildungsstätte frei zu wählen. Die Berufsausübung kann durch Gesetz +oder auf Grund eines Gesetzes geregelt werden. +(2) Niemand darf zu einer bestimmten Arbeit gezwungen werden, außer im +Rahmen einer herkömmlichen allgemeinen, für alle gleichen öffentlichen +Dienstleistungspflicht. +(3) Zwangsarbeit ist nur bei einer gerichtlich angeordneten +Freiheitsentziehung zulässig. +% +Grundgesetz Artikel 12a +(1) Männer können vom vollendeten achtzehnten Lebensjahr an zum Dienst +in den Streitkräften, im Bundesgrenzschutz oder in einem +Zivilschutzverband verpflichtet werden. +(2) Wer aus Gewissensgründen den Kriegsdienst mit der Waffe +verweigert, kann zu einem Ersatzdienst verpflichtet werden. Die Dauer +des Ersatzdienstes darf die Dauer des Wehrdienstes nicht +übersteigen. Das Nähere regelt ein Gesetz, das die Freiheit der +Gewissensentscheidung nicht beeinträchtigen darf und auch eine +Möglichkeit des Ersatzdienstes vorsehen muß, die in keinem +Zusammenhang mit den Verbänden der Streitkräfte und des +Bundesgrenzschutzes steht. +(3) Wehrpflichtige, die nicht zu einem Dienst nach Absatz 1 oder 2 +herangezogen sind, können im Verteidigungsfalle durch Gesetz oder auf +Grund eines Gesetzes zu zivilen Dienstleistungen für Zwecke der +Verteidigung einschließlich des Schutzes der Zivilbevölkerung in +Arbeitsverhältnisse verpflichtet werden; Verpflichtungen in +öffentlich-rechtliche Dienstverhältnisse sind nur zur Wahrnehmung +polizeilicher Aufgaben oder solcher hoheitlichen Aufgaben der +öffentlichen Verwaltung, die nur in einem öffentlich-rechtlichen +Dienstverhältnis erfüllt werden können, zulässig. Arbeitsverhältnisse +nach Satz 1 können bei den Streitkräften, im Bereich ihrer Versorgung +sowie bei der öffentlichen Verwaltung begründet werden; +Verpflichtungen in Arbeitsverhältnisse im Bereiche der Versorgung der +Zivilbevölkerung sind nur zulässig, um ihren lebensnotwendigen Bedarf +zu decken oder ihren Schutz sicherzustellen. +(4) Kann im Verteidigungsfalle der Bedarf an zivilen Dienstleistungen +im zivilen Sanitäts- und Heilwesen sowie in der ortsfesten +militärischen Lazarettorganisation nicht auf freiwilliger Grundlage +gedeckt werden, so können Frauen vom vollendeten achtzehnten bis zum +vollendeten fünfundfünfzigsten Lebensjahr durch Gesetz oder auf Grund +eines Gesetzes zu derartigen Dienstleistungen herangezogen werden. Sie +dürfen auf keinen Fall zum Dienst mit der Waffe verpflichtet werden. +(5) Für die Zeit vor dem Verteidigungsfalle können Verpflichtungen +nach Absatz 3 nur nach Maßgabe des Artikels 80a Abs. 1 begründet +werden. Zur Vorbereitung auf Dienstleistungen nach Absatz 3, für die +besondere Kenntnisse oder Fertigkeiten erforderlich sind, kann durch +Gesetz oder auf Grund eines Gesetzes die Teilnahme an +Ausbildungsveranstaltungen zur Pflicht gemacht werden. Satz 1 findet +insoweit keine Anwendung. +(6) Kann im Verteidigungsfalle der Bedarf an Arbeitskräften für die in +Absatz 3 Satz 2 genannten Bereiche auf freiwilliger Grundlage nicht +gedeckt werden, so kann zur Sicherung dieses Bedarfs die Freiheit der +Deutschen, die Ausübung eines Berufs oder den Arbeitsplatz aufzugeben, +durch Gesetz oder auf Grund eines Gesetzes eingeschränkt werden. Vor +Eintritt des Verteidigungsfalles gilt Absatz 5 Satz 1 entsprechend. +% +Grundgesetz Artikel 13 +(1) Die Wohnung ist unverletzlich. +(2) Durchsuchungen dürfen nur durch den Richter, bei Gefahr im Verzuge +auch durch die in den Gesetzen vorgesehenen anderen Organe angeordnet +und nur in der dort vorgeschriebenen Form durchgeführt werden. +(3) Begründen bestimmte Tatsachen den Verdacht, daß jemand eine durch +Gesetz einzeln bestimmte besonders schwere Straftat begangen hat, so +dürfen zur Verfolgung der Tat auf Grund richterlicher Anordnung +technische Mittel zur akustischen Überwachung von Wohnungen, in denen +der Beschuldigte sich vermutlich aufhält, eingesetzt werden, wenn die +Erforschung des Sachverhalts auf andere Weise unverhältnismäßig +erschwert oder aussichtslos wäre. Die Maßnahme ist zu befristen. Die +Anordnung erfolgt durch einen mit drei Richtern besetzten +Spruchkörper. Bei Gefahr im Verzuge kann sie auch durch einen +einzelnen Richter getroffen werden. +(4) Zur Abwehr dringender Gefahren für die öffentliche Sicherheit, +insbesondere einer gemeinen Gefahr oder einer Lebensgefahr, dürfen +technische Mittel zur Überwachung von Wohnungen nur auf Grund +richterlicher Anordnung eingesetzt werden. Bei Gefahr im Verzuge kann +die Maßnahme auch durch eine andere gesetzlich bestimmte Stelle +angeordnet werden; eine richterliche Entscheidung ist unverzüglich +nachzuholen. +(5) Sind technische Mittel ausschließlich zum Schutze der bei einem +Einsatz in Wohnungen tätigen Personen vorgesehen, kann die Maßnahme +durch eine gesetzlich bestimmte Stelle angeordnet werden. Eine +anderweitige Verwertung der hierbei erlangten Erkenntnisse ist nur zum +Zwecke der Strafverfolgung oder der Gefahrenabwehr und nur zulässig, +wenn zuvor die Rechtmäßigkeit der Maßnahme richterlich festgestellt +ist; bei Gefahr im Verzuge ist die richterliche Entscheidung +unverzüglich nachzuholen. +(6) Die Bundesregierung unterrichtet den Bundestag jährlich über den +nach Absatz 3 sowie über den im Zuständigkeitsbereich des Bundes nach +Absatz 4 und, soweit richterlich überprüfungsbedürftig, nach Absatz 5 +erfolgten Einsatz technischer Mittel. Ein vom Bundestag gewähltes +Gremium übt auf der Grundlage dieses Berichts die parlamentarische +Kontrolle aus. Die Länder gewährleisten eine gleichwertige +parlamentarische Kontrolle. +(7) Eingriffe und Beschränkungen dürfen im übrigen nur zur Abwehr +einer gemeinen Gefahr oder einer Lebensgefahr für einzelne Personen, +auf Grund eines Gesetzes auch zur Verhütung dringender Gefahren für +die öffentliche Sicherheit und Ordnung, insbesondere zur Behebung der +Raumnot, zur Bekämpfung von Seuchengefahr oder zum Schutze gefährdeter +Jugendlicher vorgenommen werden. +% +Grundgesetz Artikel 14 +(1) Das Eigentum und das Erbrecht werden gewährleistet. Inhalt und +Schranken werden durch die Gesetze bestimmt. +(2) Eigentum verpflichtet. Sein Gebrauch soll zugleich dem Wohle der +Allgemeinheit dienen. +(3) Eine Enteignung ist nur zum Wohle der Allgemeinheit zulässig. Sie +darf nur durch Gesetz oder auf Grund eines Gesetzes erfolgen, das Art +und Ausmaß der Entschädigung regelt. Die Entschädigung ist unter +gerechter Abwägung der Interessen der Allgemeinheit und der +Beteiligten zu bestimmen. Wegen der Höhe der Entschädigung steht im +Streitfalle der Rechtsweg vor den ordentlichen Gerichten offen. +% +Grundgesetz Artikel 15 +Grund und Boden, Naturschätze und Produktionsmittel können zum Zwecke +der Vergesellschaftung durch ein Gesetz, das Art und Ausmaß der +Entschädigung regelt, in Gemeineigentum oder in andere Formen der +Gemeinwirtschaft überführt werden. Für die Entschädigung gilt Artikel +14 Abs. 3 Satz 3 und 4 entsprechend. +% +Grundgesetz Artikel 16 +(1) Die deutsche Staatsangehörigkeit darf nicht entzogen werden. Der +Verlust der Staatsangehörigkeit darf nur auf Grund eines Gesetzes und +gegen den Willen des Betroffenen nur dann eintreten, wenn der +Betroffene dadurch nicht staatenlos wird. +(2) Kein Deutscher darf an das Ausland ausgeliefert werden. Durch +Gesetz kann eine abweichende Regelung für Auslieferungen an einen +Mitgliedstaat der Europäischen Union oder an einen internationalen +Gerichtshof getroffen werden, soweit rechtsstaatliche Grundsätze +gewahrt sind. +% +Grundgesetz Artikel 16a +(1) Politisch Verfolgte genießen Asylrecht. +(2) Auf Absatz 1 kann sich nicht berufen, wer aus einem Mitgliedstaat +der Europäischen Gemeinschaften oder aus einem anderen Drittstaat +einreist, in dem die Anwendung des Abkommens über die Rechtsstellung +der Flüchtlinge und der Konvention zum Schutze der Menschenrechte und +Grundfreiheiten sichergestellt ist. Die Staaten außerhalb der +Europäischen Gemeinschaften, auf die die Voraussetzungen des Satzes 1 +zutreffen, werden durch Gesetz, das der Zustimmung des Bundesrates +bedarf, bestimmt. In den Fällen des Satzes 1 können +aufenthaltsbeendende Maßnahmen unabhängig von einem hiergegen +eingelegten Rechtsbehelf vollzogen werden. +(3) Durch Gesetz, das der Zustimmung des Bundesrates bedarf, können +Staaten bestimmt werden, bei denen auf Grund der Rechtslage, der +Rechtsanwendung und der allgemeinen politischen Verhältnisse +gewährleistet erscheint, daß dort weder politische Verfolgung noch +unmenschliche oder erniedrigende Bestrafung oder Behandlung +stattfindet. Es wird vermutet, daß ein Ausländer aus einem solchen +Staat nicht verfolgt wird, solange er nicht Tatsachen vorträgt, die +die Annahme begründen, daß er entgegen dieser Vermutung politisch +verfolgt wird. +(4) Die Vollziehung aufenthaltsbeendender Maßnahmen wird in den Fällen +des Absatzes 3 und in anderen Fällen, die offensichtlich unbegründet +sind oder als offensichtlich unbegründet gelten, durch das Gericht nur +ausgesetzt, wenn ernstliche Zweifel an der Rechtmäßigkeit der Maßnahme +bestehen; der Prüfungsumfang kann eingeschränkt werden und verspätetes +Vorbringen unberücksichtigt bleiben. Das Nähere ist durch Gesetz zu +bestimmen. +(5) Die Absätze 1 bis 4 stehen völkerrechtlichen Verträgen von +Mitgliedstaaten der Europäischen Gemeinschaften untereinander und mit +dritten Staaten nicht entgegen, die unter Beachtung der +Verpflichtungen aus dem Abkommen über die Rechtsstellung der +Flüchtlinge und der Konvention zum Schutze der Menschenrechte und +Grundfreiheiten, deren Anwendung in den Vertragsstaaten sichergestellt +sein muß, Zuständigkeitsregelungen für die Prüfung von Asylbegehren +einschließlich der gegenseitigen Anerkennung von Asylentscheidungen +treffen. +% +Grundgesetz Artikel 17 +Jedermann hat das Recht, sich einzeln oder in Gemeinschaft mit anderen +schriftlich mit Bitten oder Beschwerden an die zuständigen Stellen und +an die Volksvertretung zu wenden. +% +Grundgesetz Artikel 17a +(1) Gesetze über Wehrdienst und Ersatzdienst können bestimmen, daß für +die Angehörigen der Streitkräfte und des Ersatzdienstes während der +Zeit des Wehr- oder Ersatzdienstes das Grundrecht, seine Meinung in +Wort, Schrift und Bild frei zu äußern und zu verbreiten (Artikel 5 +Abs. 1 Satz 1 erster Halbsatz), das Grundrecht der +Versammlungsfreiheit (Artikel 8) und das Petitionsrecht (Artikel 17), +soweit es das Recht gewährt, Bitten oder Beschwerden in Gemeinschaft +mit anderen vorzubringen, eingeschränkt werden. +(2) Gesetze, die der Verteidigung einschließlich des Schutzes der +Zivilbevölkerung dienen, können bestimmen, daß die Grundrechte der +Freizügigkeit (Artikel 11) und der Unverletzlichkeit der Wohnung +(Artikel 13) eingeschränkt werden. +% +Grundgesetz Artikel 18 +Wer die Freiheit der Meinungsäußerung, insbesondere die Pressefreiheit +(Artikel 5 Abs. 1), die Lehrfreiheit (Artikel 5 Abs. 3), die +Versammlungsfreiheit (Artikel 8), die Vereinigungsfreiheit (Artikel +9), das Brief-, Post- und Fernmeldegeheimnis (Artikel 10), das +Eigentum (Artikel 14) oder das Asylrecht (Artikel 16a) zum Kampfe +gegen die freiheitliche demokratische Grundordnung mißbraucht, +verwirkt diese Grundrechte. Die Verwirkung und ihr Ausmaß werden durch +das Bundesverfassungsgericht ausgesprochen. +% +Grundgesetz Artikel 19 +(1) Soweit nach diesem Grundgesetz ein Grundrecht durch Gesetz oder +auf Grund eines Gesetzes eingeschränkt werden kann, muß das Gesetz +allgemein und nicht nur für den Einzelfall gelten. Außerdem muß das +Gesetz das Grundrecht unter Angabe des Artikels nennen. +(2) In keinem Falle darf ein Grundrecht in seinem Wesensgehalt +angetastet werden. +(3) Die Grundrechte gelten auch für inländische juristische Personen, +soweit sie ihrem Wesen nach auf diese anwendbar sind. +(4) Wird jemand durch die öffentliche Gewalt in seinen Rechten +verletzt, so steht ihm der Rechtsweg offen. Soweit eine andere +Zuständigkeit nicht begründet ist, ist der ordentliche Rechtsweg +gegeben. Artikel 10 Abs. 2 Satz 2 bleibt unberührt. +% -- 2.11.0 From 68b15c2f4cf580abfd53721001c6890ea6a78e8e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 13:16:50 +0200 Subject: [PATCH 159/257] share/skel/dot.profile: Change fortune file to "grundrechte" Obtained from: ElectroBSD --- share/skel/dot.profile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/skel/dot.profile b/share/skel/dot.profile index ad66198ce5e8..eb53895f9097 100644 --- a/share/skel/dot.profile +++ b/share/skel/dot.profile @@ -21,4 +21,4 @@ PAGER=more; export PAGER # set ENV to a file invoked each time sh is started for interactive use. ENV=$HOME/.shrc; export ENV -if [ -x /usr/bin/fortune ] ; then /usr/bin/fortune freebsd-tips ; fi +if [ -x /usr/bin/fortune ] ; then /usr/bin/fortune grundrechte ; fi -- 2.11.0 From d2ba55fb3268c89d38cb74a99518f94ef53b59ba Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 18 Aug 2015 09:57:30 +0200 Subject: [PATCH 160/257] sys/x86: Note that ElectroBSD does not protect against compromised hosts ... if a hypervisor has been detected. Obtained from: ElectroBSD --- sys/x86/x86/identcpu.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sys/x86/x86/identcpu.c b/sys/x86/x86/identcpu.c index d4cc22363d58..8ffac953fb7e 100644 --- a/sys/x86/x86/identcpu.c +++ b/sys/x86/x86/identcpu.c @@ -2415,6 +2415,11 @@ static void print_hypervisor_info(void) { - if (*hv_vendor) - printf("Hypervisor: Origin = \"%s\"\n", hv_vendor); + if (*hv_vendor == '\0') + return; + + printf("Hypervisor: Origin = \"%s\"\n", hv_vendor); + printf("ElectroBSD security advise: It looks like you may be clown computing.\n" + "Please note that the people controlling the 'cloud' this system is\n" + "running on can monitor and sabotage everything it is doing.\n"); } -- 2.11.0 From 4cc43285bc42defe26c21ddb6f4addaa5fa91e0f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 8 Jan 2017 23:11:47 +0100 Subject: [PATCH 161/257] Detach enigma(1) from the build Obtained from: ElectroBSD --- usr.bin/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/usr.bin/Makefile b/usr.bin/Makefile index b87c26a6b7b0..d0c9c43b1882 100644 --- a/usr.bin/Makefile +++ b/usr.bin/Makefile @@ -40,7 +40,6 @@ SUBDIR= alias \ du \ elf2aout \ elfdump \ - enigma \ env \ expand \ false \ -- 2.11.0 From 2a4362f447231d02f338e10d4a7e82fa2462f234 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 8 Jan 2017 23:42:58 +0100 Subject: [PATCH 162/257] Detach send-pr from the build Obtained from: ElectroBSD --- usr.bin/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/usr.bin/Makefile b/usr.bin/Makefile index d0c9c43b1882..928a10acb77b 100644 --- a/usr.bin/Makefile +++ b/usr.bin/Makefile @@ -137,7 +137,6 @@ SUBDIR= alias \ script \ sdiff \ sed \ - send-pr \ seq \ shar \ showmount \ -- 2.11.0 From 13f9812e39edafb011d1e001815740145e49fde3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 22 Sep 2015 12:34:53 +0200 Subject: [PATCH 163/257] sys/vm: vm_pageout_scan: Stop pass after reaching the "maximum" value ... explicitly understood by vm_pageout_scan(). Should prevent (purely cosmetic) issues like: fk@r500 ~ $sudo /usr/src/share/dtrace/monitor-page-scanner [...] 2015 Sep 22 12:15:54: Scan goal 59: Invalid 2015 Sep 22 12:15:54: Scan goal 60: Invalid 2015 Sep 22 12:15:55: Scan goal 61: Invalid 2015 Sep 22 12:15:55: Scan goal 62: Invalid 2015 Sep 22 12:15:55: Scan goals in the previous minute: Launder dirty pages 1 Pageout dirty pages 1 Move inactive to cache or free 2 Invalid 5 Update active LRU/deactivate pages 28 2015 Sep 22 12:15:55: Seconds since last 'Move inactive to cache or free' pass: 30 2015 Sep 22 12:15:55: Seconds since last 'Launder dirty pages' pass: 30 2015 Sep 22 12:15:55: Seconds since last 'Pageout dirty pages' pass: 30 Obtained from: ElectroBSD --- sys/vm/vm_pageout.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index a3f8774c9160..67cf415d2e1e 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -839,6 +839,7 @@ unlock_mp: return (error); } +#define VMD_PASS_MAX 3 /* * vm_pageout_scan does the dirty work for the pageout daemon. * @@ -861,6 +862,9 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) int starting_page_shortage, vnodes_skipped; boolean_t pageout_ok, queue_locked; + KASSERT(pass <= VMD_PASS_MAX, + ("vm_pageout_scan: Invalid pass code %d", pass)); + /* * If we need to reclaim memory ask kernel caches to return * some. We rate limit to avoid thrashing. @@ -1577,7 +1581,8 @@ vm_pageout_worker(void *arg) mtx_unlock(&vm_page_queue_free_mtx); if (pass > 1) pause("psleep", hz / 2); - pass++; + if (pass < VMD_PASS_MAX) + pass++; } else { /* * Yes. Sleep until pages need to be reclaimed or -- 2.11.0 From 9243b0b35cd64296adfa350a6a47e17faebed887 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 19:10:51 +0200 Subject: [PATCH 164/257] share/dtrace: Add monitor-page-scanner ... which, who would have guessed it, monitors the vm page scanner. It's useful to tune the sysctls for the inactive page limit. Obtained from: ElectroBSD --- share/dtrace/monitor-page-scanner | 168 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100755 share/dtrace/monitor-page-scanner diff --git a/share/dtrace/monitor-page-scanner b/share/dtrace/monitor-page-scanner new file mode 100755 index 000000000000..85537caff8bf --- /dev/null +++ b/share/dtrace/monitor-page-scanner @@ -0,0 +1,168 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * monitor-page-scanner + * + * Traces the vm page scanner. + * + * Relies on SDT probes that currrently are not part of vanilla FreeBSD. + * + * Copyright (c) 2015 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet +#pragma D option dynvarsize=10m + +dtrace:::BEGIN +{ + goal[0] = "Update active LRU/deactivate pages"; + goal[1] = "Move inactive to cache or free"; + goal[2] = "Launder dirty pages"; + goal[3] = "Pageout dirty pages"; + start_time = walltimestamp; + + min_pass = 2; + + printf("%Y: Monitoring the page scanner. Minimum pass value to show 'boring' scans ", walltimestamp); + printf("without memory pressure or inactive page surplus: %d (%s). Press CTRL-C to abort.\n", + min_pass, goal[min_pass]); +} + +vm:kernel::checked-inactive-pages +{ + this->to_free = (int)arg0; +} + +vm:kernel::checked-inactive-pages +/this->to_free/ +{ + this->above_limit = (int)arg1; + + printf("%Y: %s: Inactive page surplus: %d, above limit: %d.\n", + walltimestamp, probename, this->to_free, this->above_limit); +} + +vm:kernel::before-inactive-scan +{ + this->pass = (int)arg1; + this->page_shortage = (int)arg2; + this->inactive_page_surplus = (int)arg4; + + /* + * Every pass code above 3 is treated like 3, + * adjust index accordingly. + */ + this->goal_index = (this->pass <= 3) ? this->pass : 3; + this->goal = goal[this->goal_index]; + @goals[this->goal] = count(); + @goals_total[this->goal] = count(); + last_pass[this->pass] = timestamp; +} + +vm:kernel::before-inactive-scan +/(this->pass >= min_pass) || (this->page_shortage > 0) +|| (this->inactive_page_surplus > 0)/ +{ + /* this->vmd = (struct vm_domain *)arg0; */ + this->deficit = (int)arg3; + + printf("%Y: %s: Scan goal %d: %s.\n", + walltimestamp, probename, this->pass, this->goal); + printf("%Y: %s: Page shortage: %d, inactive page surplus: %d\n", + walltimestamp, probename, this->page_shortage, this->inactive_page_surplus); + printf("%Y: v_free_target: %d, v_free_count: %d, v_cache_count: %d.\n", + walltimestamp, `vm_cnt.v_free_target, `vm_cnt.v_free_count, `vm_cnt.v_cache_count); + printf("%Y: v_inactive_target: %d. v_inactive_count: %d. deficit: %d\n", + walltimestamp, `vm_cnt.v_inactive_target, `vm_cnt.v_inactive_count, this->deficit); +} + +vm:kernel::after-inactive-scan +{ + this->pass = (int)arg1; + this->page_shortage = (int)arg2; + this->addl_page_shortage = (int)arg3; + this->vnodes_skipped = (int)arg4; + this->inactive_page_surplus = (int)arg5; +} + +vm:kernel::after-inactive-scan +/(this->pass >= min_pass) || (this->page_shortage > 0) || +(this->addl_page_shortage > 0) || (this->vnodes_skipped > 0) || +(this->inactive_page_surplus > 0)/ +{ + printf("%Y: %s pass %d: page shortage: %d, inactive page surplus: %d, addl shortage: %d, vnodes skipped: %d.\n", + walltimestamp, probename, this->pass, this->page_shortage, + this->inactive_page_surplus, this->addl_page_shortage, this->vnodes_skipped); +} + +vm:kernel::before-active-scan, +vm:kernel::after-active-scan +{ + this->pass = (int)arg1; + this->page_shortage = (int)arg2; +} + +vm:kernel::before-active-scan, +vm:kernel::after-active-scan +/(this->pass >= min_pass) || (this->page_shortage > 0)/ +{ + printf("%Y: %s pass %d: page_shortage: %d.\n", + walltimestamp, probename, this->pass, this->page_shortage); +} + +tick-60s +{ + printf("%Y: Scan goals in the previous minute:", walltimestamp); + printa(@goals); + trunc(@goals); +} + +tick-60s +/last_pass[1]/ +{ + this->pass = 1; + this->elapsed = (timestamp - last_pass[this->pass]) / 1000000000; + printf("%Y: Seconds since last '%s' pass: %d.\n", + walltimestamp, goal[this->pass], this->elapsed); +} + +tick-60s +/last_pass[2]/ +{ + this->pass = 2; + this->elapsed = (timestamp - last_pass[this->pass]) / 1000000000; + printf("%Y: Seconds since last '%s' pass: %d.\n", + walltimestamp, goal[this->pass], this->elapsed); +} + +tick-60s +/last_pass[3]/ +{ + this->pass = 3; + this->elapsed = (timestamp - last_pass[this->pass]) / 1000000000; + printf("%Y: Seconds since last '%s' pass: %d.\n", + walltimestamp, goal[this->pass], this->elapsed); +} + +END +{ + printf("%Y: Scan goals since start of script at %Y:", + walltimestamp, start_time); + printa(@goals_total); + + /* Clear aggregates so DTrace does not show them again. */ + trunc(@goals_total); + trunc(@goals); +} -- 2.11.0 From 2b7b93635ed90c34a7ffeff08b32b04e04f7f7cd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 13 Oct 2015 18:19:42 +0200 Subject: [PATCH 165/257] libdtrace: Replace another timestamp related assert() with a warning It triggered a lot less often, but was still annoying. Example: 2015 Oct 13 18:01:06: v_free_target: 20888, v_free_count: 44001, v_cache_count: 0. 2015 Oct 13 18:01:06: v_inactive_target: 31332. v_inactive_count: 364311. deficit: 0 2015 Oct 13 18:01:06: after-inactive-scan pass 3: page shortage: -23113, inactive page surplus: 10000, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:06: before-active-scan pass 3: page_shortage: -356092. 2015 Oct 13 18:01:06: after-active-scan pass 3: page_shortage: -356092. 2015 Oct 13 18:01:06: checked-inactive-pages: Inactive page surplus: 10000, above limit: 252979. dtrace: cpu clocks out of sync (18128465664478 < 18128471798881; offset: 6134403). Results may be incorrect! dtrace: cpu clocks out of sync (18129463671844 < 18129469778266; offset: 6106422). Results may be incorrect! 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253446. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253446. 2015 Oct 13 18:01:07: before-inactive-scan: Scan goal 1: Move inactive to cache or free. 2015 Oct 13 18:01:07: before-inactive-scan: Page shortage: -37096, inactive page surplus: 10000 2015 Oct 13 18:01:07: v_free_target: 20888, v_free_count: 57984, v_cache_count: 0. 2015 Oct 13 18:01:07: v_inactive_target: 31332. v_inactive_count: 364778. deficit: 0 2015 Oct 13 18:01:07: after-inactive-scan pass 1: page shortage: -37203, inactive page surplus: 9893, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253250. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253250. 2015 Oct 13 18:01:07: before-inactive-scan: Scan goal 2: Launder dirty pages. 2015 Oct 13 18:01:07: before-inactive-scan: Page shortage: -37191, inactive page surplus: 10000 2015 Oct 13 18:01:07: v_free_target: 20888, v_free_count: 58079, v_cache_count: 0. 2015 Oct 13 18:01:07: v_inactive_target: 31332. v_inactive_count: 364582. deficit: 0 2015 Oct 13 18:01:07: after-inactive-scan pass 2: page shortage: -37194, inactive page surplus: 9997, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:07: before-active-scan pass 2: page_shortage: -370441. 2015 Oct 13 18:01:07: after-active-scan pass 2: page_shortage: -370441. 2015 Oct 13 18:01:07: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253268. dtrace: cpu clocks out of sync (18130462652022 < 18130468761237; offset: 6109215). Results may be incorrect! 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253331. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253331. 2015 Oct 13 18:01:08: before-inactive-scan: Scan goal 1: Move inactive to cache or free. 2015 Oct 13 18:01:08: before-inactive-scan: Page shortage: -34625, inactive page surplus: 10000 2015 Oct 13 18:01:08: v_free_target: 20888, v_free_count: 55513, v_cache_count: 0. 2015 Oct 13 18:01:08: v_inactive_target: 31332. v_inactive_count: 364663. deficit: 0 2015 Oct 13 18:01:08: after-inactive-scan pass 1: page shortage: -34645, inactive page surplus: 9980, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253286. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253286. 2015 Oct 13 18:01:08: before-inactive-scan: Scan goal 2: Launder dirty pages. 2015 Oct 13 18:01:08: before-inactive-scan: Page shortage: -34395, inactive page surplus: 10000 2015 Oct 13 18:01:08: v_free_target: 20888, v_free_count: 55283, v_cache_count: 0. 2015 Oct 13 18:01:08: v_inactive_target: 31332. v_inactive_count: 364618. deficit: 0 2015 Oct 13 18:01:08: after-inactive-scan pass 2: page shortage: -34396, inactive page surplus: 9999, addl shortage: 0, vnodes skipped: 0. 2015 Oct 13 18:01:08: before-active-scan pass 2: page_shortage: -367391. 2015 Oct 13 18:01:08: after-active-scan pass 2: page_shortage: -367391. 2015 Oct 13 18:01:08: checked-inactive-pages: Inactive page surplus: 10000, above limit: 253285. 2015 Oct 13 18:01:09: checked-inactive-pages: Inactive page surplus: 10000, above limit: 256158. 2015 Oct 13 18:01:09: before-inactive-scan: Scan goal 3: Pageout dirty pages. 2015 Oct 13 18:01:09: before-inactive-scan: Page shortage: -34665, inactive page surplus: 10000 2015 Oct 13 18:01:09: v_free_target: 20888, v_free_count: 55553, v_cache_count: 0. 2015 Oct 13 18:01:09: v_inactive_target: 31332. v_inactive_count: 367490. deficit: 0 dtrace: cpu clocks out of sync (18131462608939 < 18131468746340; offset: 6137401). Results may be incorrect! Assertion failed: (timestamp >= dtp->dt_last_timestamp), file /usr/src/cddl/lib/libdtrace/../../../cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c, line 3352. Obtained from: ElectroBSD --- cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c index 39530cf41e83..3b66c3522cdc 100644 --- a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c +++ b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c @@ -3349,7 +3349,13 @@ dtrace_consume(dtrace_hdl_t *dtp, FILE *fp, break; timestamp = dt_buf_oldest(buf, dtp); - assert(timestamp >= dtp->dt_last_timestamp); + if (timestamp < dtp->dt_last_timestamp) { + warnx("cpu clocks out of sync " + "(%ju < %ju; offset: %ju). " + "Results may be incorrect!", + timestamp, dtp->dt_last_timestamp, + dtp->dt_last_timestamp - timestamp); + } dtp->dt_last_timestamp = timestamp; if (timestamp == buf->dtbd_timestamp) { -- 2.11.0 From 04359ba6ea3e90664d5dfc00d676824e03e37556 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Sep 2015 13:49:38 +0200 Subject: [PATCH 166/257] sys/modules: Don't (try to) build qlxbge if the user objects to binary blobs The module depends on source-less microcode in sys/dev/qlxgbe/ql_fw.c. Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=204749 PR submission date: 2015-11-22 --- sys/modules/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 0795a902a43e..11a9e3d3a7e0 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -682,7 +682,9 @@ _ntb= ntb _pms= pms _qlxge= qlxge _qlxgb= qlxgb +.if ${MK_SOURCELESS_UCODE} != "no" _qlxgbe= qlxgbe +.endif _sfxge= sfxge .if ${MK_BHYVE} != "no" || defined(ALL_MODULES) -- 2.11.0 From df47bf62a875b03124329fd584facb62bf240c67 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Sep 2015 12:52:29 +0200 Subject: [PATCH 167/257] sys/modules: Don't (try to) build bxe if the user objects to binary blobs The module contains source-less microcode: sys/dev/bxe/5771[0-2]_init_values.c Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=204747 PR submission date: 2015-11-22 --- sys/modules/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 11a9e3d3a7e0..eafbe5bcf7ba 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -543,7 +543,9 @@ _an= an _aout= aout _bios= bios _bktr= bktr +.if ${MK_SOURCELESS_UCODE} != "no" _bxe= bxe +.endif _cardbus= cardbus _cbb= cbb _cpuctl= cpuctl -- 2.11.0 From 367bbe6824898610298a21b759356baa2aa56559 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 27 Nov 2015 00:20:59 +0100 Subject: [PATCH 168/257] etc: Register uid and gid for Tor ... so the user doesn't have to do it manually on the host system after installing Tor in a jail. Obtained from: ElectroBSD --- etc/group | 1 + etc/master.passwd | 1 + 2 files changed, 2 insertions(+) diff --git a/etc/group b/etc/group index 3136230b0790..49eccdcb8dac 100644 --- a/etc/group +++ b/etc/group @@ -30,6 +30,7 @@ network:*:69: audit:*:77: www:*:80: _ypldap:*:160: +_tor:*:256: hast:*:845: ggatec:*:846: ggated:*:847: diff --git a/etc/master.passwd b/etc/master.passwd index c6005dc66899..b4a4a9b23f0e 100644 --- a/etc/master.passwd +++ b/etc/master.passwd @@ -23,6 +23,7 @@ pop:*:68:6::0:0:Post Office Owner:/nonexistent:/usr/sbin/nologin auditdistd:*:78:77::0:0:Auditdistd unprivileged user:/var/empty:/usr/sbin/nologin www:*:80:80::0:0:World Wide Web Owner:/nonexistent:/usr/sbin/nologin _ypldap:*:160:160::0:0:YP LDAP unprivileged user:/var/empty:/usr/sbin/nologin +_tor:*:256:256::0:0:Onion delivery agent:/nonexistent:/usr/sbin/nologin hast:*:845:845::0:0:HAST unprivileged user:/var/empty:/usr/sbin/nologin ggatec:*:846:846::0:0:ggatec unprivileged user:/var/empty:/usr/sbin/nologin ggated:*:847:847::0:0:ggated unprivileged user:/var/empty:/usr/sbin/nologin -- 2.11.0 From 0cdd4a2c98d0c9a1c4cb1716e30042925d1a4f61 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 9 Jan 2016 00:37:52 +0100 Subject: [PATCH 169/257] crontab: Default to not sending out (most-likely unencrypted) mails Obtained from: ElectroBSD --- etc/crontab | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etc/crontab b/etc/crontab index e1e6e8862c8d..22204f81a0fd 100644 --- a/etc/crontab +++ b/etc/crontab @@ -1,9 +1,8 @@ -# /etc/crontab - root's crontab for FreeBSD -# -# $FreeBSD$ +# /etc/crontab - root's crontab for ElectroBSD # SHELL=/bin/sh PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin +MAILTO="" # #minute hour mday month wday who command # -- 2.11.0 From 76a2d133103dda3fa41e085c16b7c28293d8a078 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 16 Jan 2016 15:24:51 +0100 Subject: [PATCH 170/257] Detach unreproducible uzip tests (added in r293821) that aren't relevant for ElectroBSD Obtained from: ElectroBSD --- tests/sys/geom/class/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/sys/geom/class/Makefile b/tests/sys/geom/class/Makefile index b633fd081ee1..92c9b9f8e38f 100644 --- a/tests/sys/geom/class/Makefile +++ b/tests/sys/geom/class/Makefile @@ -14,7 +14,8 @@ TESTS_SUBDIRS+= nop TESTS_SUBDIRS+= raid3 TESTS_SUBDIRS+= shsec TESTS_SUBDIRS+= stripe -TESTS_SUBDIRS+= uzip +# Currently not reproducible +#TESTS_SUBDIRS+= uzip ${PACKAGE}FILES+= geom_subr.sh -- 2.11.0 From 6cb19c83e62148378afb45b6a65d86a18422c31f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 21 Jan 2016 12:49:29 +0100 Subject: [PATCH 171/257] OpenSSH: Disable DSA keys by default ... as it's done by vanilla OpenSSH since 6.9p1. This reverts FreeBSD r294495 which enabled them again due to POLA concerns that aren't relevant for ElectroBSD. Obtained from: ElectroBSD --- UPDATING | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/UPDATING b/UPDATING index b9e1e1770a1c..09ea202f7463 100644 --- a/UPDATING +++ b/UPDATING @@ -182,6 +182,10 @@ from older version of current across the gcc/clang cutover is a bit fragile. restored by setting vfs.zfs.vol.recursive=1. 20160119: + The default configuration of ssh(1) no longer allows to use ssh-dss + keys. To enable using them, add 'ssh-dss' to PubkeyAcceptedKeyTypes + option in the /etc/ssh/ssh_config. Refer to ssh_config(5) for more + information. The NONE and HPN patches has been removed from OpenSSH. They are still available in the security/openssh-portable port. -- 2.11.0 From 6a0d17a794a39edc2c5eafcc5f1d48f678d6d7e9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Mar 2016 14:25:31 +0100 Subject: [PATCH 172/257] OpenSSH: Disable tcp wrapper support Obtained from: ElectroBSD --- crypto/openssh/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/openssh/config.h b/crypto/openssh/config.h index 364ebe33e2ef..a26c7647c913 100644 --- a/crypto/openssh/config.h +++ b/crypto/openssh/config.h @@ -1408,7 +1408,7 @@ /* #undef LASTLOG_WRITE_PUTUTXLINE */ /* Define if you want TCP Wrappers support */ -#define LIBWRAP 1 +/* #undef LIBWRAP 1 */ /* Define to whatever link() returns for "not supported" if it doesn't return EOPNOTSUPP. */ -- 2.11.0 From 81be2a8a238ce18c6c2a243bfa08e038d510b811 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Mar 2016 14:27:33 +0100 Subject: [PATCH 173/257] OpenSSH: Remove AES-CBC ciphers from the default cipher list on the server As it's done in OpenSSH 6.7p1 and later. This reverts FreeBSD r296634. Obtained from: ElectroBSD --- crypto/openssh/FREEBSD-upgrade | 8 +------- crypto/openssh/myproposal.h | 5 ++--- crypto/openssh/sshd_config.5 | 3 +-- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/crypto/openssh/FREEBSD-upgrade b/crypto/openssh/FREEBSD-upgrade index 4b31eb3532d6..8161baddaf5d 100644 --- a/crypto/openssh/FREEBSD-upgrade +++ b/crypto/openssh/FREEBSD-upgrade @@ -1,3 +1,4 @@ + FreeBSD maintainer's guide to OpenSSH-portable ============================================== @@ -160,13 +161,6 @@ ignore HPN-related configuration options to avoid breaking existing configurations. -9) AES-CBC - - The AES-CBC ciphers were removed from the server-side proposal list - in 6.7p1 due to theoretical weaknesses and the availability of - superior ciphers (including AES-CTR and AES-GCM). We have re-added - them for compatibility with third-party clients. - This port was brought to you by (in no particular order) DARPA, NAI diff --git a/crypto/openssh/myproposal.h b/crypto/openssh/myproposal.h index 68d29e86b63e..2af5c47cf82a 100644 --- a/crypto/openssh/myproposal.h +++ b/crypto/openssh/myproposal.h @@ -111,11 +111,10 @@ #define KEX_SERVER_ENCRYPT \ "chacha20-poly1305@openssh.com," \ "aes128-ctr,aes192-ctr,aes256-ctr" \ - AESGCM_CIPHER_MODES \ - ",aes128-cbc,aes192-cbc,aes256-cbc" + AESGCM_CIPHER_MODES #define KEX_CLIENT_ENCRYPT KEX_SERVER_ENCRYPT "," \ - "3des-cbc" + "aes128-cbc,aes192-cbc,aes256-cbc,3des-cbc" #define KEX_SERVER_MAC \ "umac-64-etm@openssh.com," \ diff --git a/crypto/openssh/sshd_config.5 b/crypto/openssh/sshd_config.5 index 1359a20bced9..311abaef8191 100644 --- a/crypto/openssh/sshd_config.5 +++ b/crypto/openssh/sshd_config.5 @@ -482,8 +482,7 @@ The default is: .Bd -literal -offset indent chacha20-poly1305@openssh.com, aes128-ctr,aes192-ctr,aes256-ctr, -aes128-gcm@openssh.com,aes256-gcm@openssh.com, -aes128-cbc,aes192-cbc,aes256-cbc +aes128-gcm@openssh.com,aes256-gcm@openssh.com .Ed .Pp The list of available ciphers may also be obtained using the -- 2.11.0 From 4e4f9fb8f1f250d7c332bc1c6721d3fcb3de50ab Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Mar 2016 14:53:42 +0100 Subject: [PATCH 174/257] sshd: Disable UsePam, X11Forwarding and UseDNS by default Obtained from: ElectroBSD --- crypto/openssh/servconf.c | 6 +++--- crypto/openssh/sshd_config | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/openssh/servconf.c b/crypto/openssh/servconf.c index 66d24cefd4e2..9844248ef51c 100644 --- a/crypto/openssh/servconf.c +++ b/crypto/openssh/servconf.c @@ -203,7 +203,7 @@ fill_default_server_options(ServerOptions *options) /* Portable-specific options */ if (options->use_pam == -1) - options->use_pam = 1; + options->use_pam = 0; /* Standard Options */ if (options->protocol == SSH_PROTO_UNKNOWN) @@ -256,7 +256,7 @@ fill_default_server_options(ServerOptions *options) if (options->print_lastlog == -1) options->print_lastlog = 1; if (options->x11_forwarding == -1) - options->x11_forwarding = 1; + options->x11_forwarding = 0; if (options->x11_display_offset == -1) options->x11_display_offset = 10; if (options->x11_use_localhost == -1) @@ -336,7 +336,7 @@ fill_default_server_options(ServerOptions *options) if (options->max_sessions == -1) options->max_sessions = DEFAULT_SESSIONS_MAX; if (options->use_dns == -1) - options->use_dns = 1; + options->use_dns = 0; if (options->client_alive_interval == -1) options->client_alive_interval = 0; if (options->client_alive_count_max == -1) diff --git a/crypto/openssh/sshd_config b/crypto/openssh/sshd_config index 075857c354dc..3c6147c2d53a 100644 --- a/crypto/openssh/sshd_config +++ b/crypto/openssh/sshd_config @@ -97,12 +97,12 @@ # If you just want the PAM account and session checks to run without # PAM authentication, then enable this but set PasswordAuthentication # and ChallengeResponseAuthentication to 'no'. -#UsePAM yes +#UsePAM no #AllowAgentForwarding yes #AllowTcpForwarding yes #GatewayPorts no -#X11Forwarding yes +#X11Forwarding no #X11DisplayOffset 10 #X11UseLocalhost yes #PermitTTY yes @@ -115,7 +115,7 @@ #Compression delayed #ClientAliveInterval 0 #ClientAliveCountMax 3 -#UseDNS yes +#UseDNS no #PidFile /var/run/sshd.pid #MaxStartups 10:30:100 #PermitTunnel no -- 2.11.0 From 4f94d0f19a71cb4e1fcd0763d8883b35b4b8d6cb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Mar 2016 14:42:00 +0100 Subject: [PATCH 175/257] OpenSSH ssh and sshd: Default to not sending an VersionAddendum This is also the default used by OpenSSH upstream. Obtained from: ElectroBSD --- crypto/openssh/ssh_config | 2 +- crypto/openssh/sshd_config | 2 +- crypto/openssh/version.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/openssh/ssh_config b/crypto/openssh/ssh_config index 54dffac92e77..d7df585bcdfd 100644 --- a/crypto/openssh/ssh_config +++ b/crypto/openssh/ssh_config @@ -50,4 +50,4 @@ # ProxyCommand ssh -q -W %h:%p gateway.example.com # RekeyLimit 1G 1h # VerifyHostKeyDNS yes -# VersionAddendum FreeBSD-20161230 +# VersionAddendum none diff --git a/crypto/openssh/sshd_config b/crypto/openssh/sshd_config index 3c6147c2d53a..94b94e8b5d6e 100644 --- a/crypto/openssh/sshd_config +++ b/crypto/openssh/sshd_config @@ -121,7 +121,7 @@ #PermitTunnel no #ChrootDirectory none #UseBlacklist no -#VersionAddendum FreeBSD-20161230 +#VersionAddendum none # no default banner path #Banner none diff --git a/crypto/openssh/version.h b/crypto/openssh/version.h index 540aad8f5df1..4068d1775594 100644 --- a/crypto/openssh/version.h +++ b/crypto/openssh/version.h @@ -6,7 +6,7 @@ #define SSH_PORTABLE "p2" #define SSH_RELEASE SSH_VERSION SSH_PORTABLE -#define SSH_VERSION_FREEBSD "FreeBSD-20161230" +#define SSH_VERSION_FREEBSD "" #ifdef WITH_OPENSSL #define OPENSSL_VERSION SSLeay_version(SSLEAY_VERSION) -- 2.11.0 From 44b0968c0011cf44da10630b850425c58cedacf0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 2 Feb 2016 20:28:35 +0100 Subject: [PATCH 176/257] release/amd64/make-memstick.sh: Use the shiny new -T switch to clamp the remaining timestamps Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 66ee073c63ba..4263855e011d 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -37,7 +37,11 @@ echo 'root_rw_mount="NO"' > ${1}/etc/rc.conf.local mtree -c -k time -p "${1}" | sed \ -e "s@time=.*@time=${EPOCH_DATE-0}.000000000 uname=root gname=wheel@" \ > "${2}.mtree" || return 1 -makefs -B little -o label="${VOLUME_LABEL}" -F "${2}.mtree" ${2}.part ${1} + +# The base makefs may not have -T support yet, +# thus we use the fresh one that goes into the image. +dist/base/usr/sbin/makefs -T "${EPOCH_DATE-0}" \ + -B little -o label="${VOLUME_LABEL}" -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" exit 1 -- 2.11.0 From 44c4ea6ec91810253c8a56df2021fa5d397fc32c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Feb 2016 15:38:15 +0100 Subject: [PATCH 177/257] release/scripts/strip-freebsd.sh: Add riscv to the list of unsupported architectures Obtained from: ElectroBSD --- release/scripts/strip-freebsd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh index 56be187cca4c..9e49e1fa8cca 100755 --- a/release/scripts/strip-freebsd.sh +++ b/release/scripts/strip-freebsd.sh @@ -74,7 +74,7 @@ get_files_to_ditch() { # source tarball and to reduce the number of files that # should be audited for license and security issues. get_unsupported_architectures() { - echo "arm arm64 mips pc98 powerpc sparc64" + echo "arm arm64 mips pc98 powerpc riscv sparc64" } # These depend on or contain proprietary firmware that is included in sys/contrib/dev -- 2.11.0 From 103330f0d7d5a71f9c586e0d847d8093f9ed6d26 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Feb 2016 15:49:00 +0100 Subject: [PATCH 178/257] strip-freebsd.sh: Delete sys/netnatm and sys/ofed Obtained from: ElectroBSD --- release/scripts/strip-freebsd.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh index 9e49e1fa8cca..fea0889485a6 100755 --- a/release/scripts/strip-freebsd.sh +++ b/release/scripts/strip-freebsd.sh @@ -97,6 +97,11 @@ get_unused_contrib_dirs() { echo "apr apr-util ipfilter ofed sendmail serf subversion tcsh" } +# Only includes directory that aren't architecture-specific +get_unused_cross_platform_sys_dirs() { + echo "netnatm ofed" +} + get_directories_to_ditch() { local arch \ dir arch_dir sys_contrib contrib_dir @@ -110,6 +115,13 @@ get_directories_to_ditch() { done done + for dir in $(get_unused_cross_platform_sys_dirs); do + potential_directory="sys/${dir}" + if [ -d "${potential_directory}" ]; then + echo "${potential_directory}" + fi + done + for sys_contrib in ipfilter octeon-sdk; do echo "sys/contrib/${sys_contrib}" done -- 2.11.0 From 5b9fbfa4248657177e124d162cafe336ac5992b2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 22 Feb 2016 13:15:19 +0100 Subject: [PATCH 179/257] include/Makefile: Unbreak build without atm sources Obtained from: ElectroBSD --- include/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/Makefile b/include/Makefile index 2fb8e3ca6e8a..48f9f653f932 100644 --- a/include/Makefile +++ b/include/Makefile @@ -40,7 +40,7 @@ LHDRS= aio.h errno.h fcntl.h linker_set.h poll.h stdatomic.h stdint.h \ syslog.h ucontext.h LDIRS= bsm cam geom net net80211 netgraph netinet netinet6 \ - netipsec netnatm netsmb nfs nfsclient nfsserver sys vm + netipsec netsmb nfs nfsclient nfsserver sys vm LSUBDIRS= cam/ata cam/nvme cam/scsi \ dev/acpica dev/agp dev/an dev/bktr dev/ciss dev/filemon dev/firewire \ @@ -54,7 +54,7 @@ LSUBDIRS= cam/ata cam/nvme cam/scsi \ geom/mirror geom/mountver geom/multipath geom/nop \ geom/raid geom/raid3 geom/shsec geom/stripe geom/virstor \ net/altq \ - netgraph/atm netgraph/netflow \ + netgraph/netflow \ netinet/cc \ security/audit \ security/mac_biba security/mac_bsdextended security/mac_lomac \ -- 2.11.0 From f84d4de01eca8b5abb8b069de57cb9896e5e6e04 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 22 Feb 2016 16:24:43 +0100 Subject: [PATCH 180/257] sys/modules: Skip mlx5 when building without OFED Fixes: ===> mlx (depend) machine -> /usr/src/sys/amd64/include x86 -> /usr/src/sys/x86/include awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/device_if.m -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/bus_if.m -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/dev/pci/pci_if.m -h rm -f .depend CC='cc' mkdep -f .depend -a -nostdinc -D_KERNEL -DKLD_MODULE -DHAVE_KERNEL_OPTION_HEADERS -I. -I/usr/src/sys -I/usr/obj/usr/src/sys/ELECTRO_BLOAT -D__printf__=__freebsd_kprintf__ -std=iso9899:1999 -include /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_global.h /usr/src/sys/modules/mlx/../../dev/mlx/mlx.c /usr/src/sys/modules/mlx/../../dev/mlx/mlx_pci.c /usr/src/sys/modules/mlx/../../dev/mlx/mlx_disk.c ===> mlx5 (depend) machine -> /usr/src/sys/amd64/include x86 -> /usr/src/sys/x86/include awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/device_if.m -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/bus_if.m -h awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -p awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -q awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/dev/pci/pci_if.m -h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_inet.h opt_inet.h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_inet6.h opt_inet6.h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_random.h opt_random.h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_rss.h opt_rss.h rm -f .depend CC='cc' mkdep -f .depend -a -nostdinc -D_KERNEL -DKLD_MODULE -I/usr/src/sys/modules/mlx5/../../ofed/include -I/usr/src/sys/modules/mlx5/../../compat/linuxkpi/common/include -DHAVE_KERNEL_OPTION_HEADERS -I. -I /usr/src/sys -I/usr/obj/usr/src/sys/ELECTRO_BLOAT -D__printf__=__freebsd_kprintf__ -std=iso9899:1999 -include /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_global.h /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core /mlx5_alloc.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_cmd.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_cq.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_eq.c /usr/src/sys /modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_eswitch_vacl.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_flow_table.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_fw.c /usr/src/sys/modules/ mlx5/../../dev/mlx5/mlx5_core/mlx5_health.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_mad.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_main.c /usr/src/sys/modules/mlx5/../../dev/mlx5/ mlx5_core/mlx5_mcg.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_mr.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_pagealloc.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_pd.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_port.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_qp.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_srq.c /usr/src/sys/modules/ml x5/../../dev/mlx5/mlx5_core/mlx5_transobj.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_uar.c /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_vport.c /usr/src/sys/modules/mlx5/../../dev/mlx5 /mlx5_core/mlx5_wq.c In file included from /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_alloc.c:33: In file included from /usr/src/sys/dev/mlx5/driver.h:41: /usr/src/sys/dev/mlx5/device.h:32:10: fatal error: 'rdma/ib_verbs.h' file not found ^ 1 error generated. In file included from /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_cmd.c:38: In file included from /usr/src/sys/dev/mlx5/driver.h:41: /usr/src/sys/dev/mlx5/device.h:32:10: fatal error: 'rdma/ib_verbs.h' file not found ^ 1 error generated. In file included from /usr/src/sys/modules/mlx5/../../dev/mlx5/mlx5_core/mlx5_cq.c:31: In file included from /usr/src/sys/dev/mlx5/driver.h:41: /usr/src/sys/dev/mlx5/device.h:32:10: fatal error: 'rdma/ib_verbs.h' file not found ^ 1 error generated. [...] Obtained from: ElectroBSD --- sys/modules/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index eafbe5bcf7ba..8a4f794c6132 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -639,8 +639,8 @@ _iwnfw= iwnfw _mlx4= mlx4 _mlx4ib= mlx4ib _mlxen= mlxen -.endif _mlx5= mlx5 +.endif .if (${MK_INET_SUPPORT} != "no" && ${MK_INET6_SUPPORT} != "no") || \ defined(ALL_MODULES) _mlx5en= mlx5en -- 2.11.0 From f4e0ce5420cc1f22443a0999f1df53ff33c4a355 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 22 Feb 2016 17:03:33 +0100 Subject: [PATCH 181/257] sys/modules: Skip mlx5en when building without OFED Fixes: ===> mlx5en (all) machine -> /usr/src/sys/amd64/include x86 -> /usr/src/sys/x86/include awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/device_if.m -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/kern/bus_if.m -h awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -p awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -q awk -f /usr/src/sys/tools/vnode_if.awk /usr/src/sys/kern/vnode_if.src -h awk -f /usr/src/sys/tools/makeobjops.awk /usr/src/sys/dev/pci/pci_if.m -h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_inet.h opt_inet.h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_inet6.h opt_inet6.h ln -sf /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_rss.h opt_rss.h cc -O2 -pipe -fno-strict-aliasing -Werror -D_KERNEL -DKLD_MODULE -nostdinc -I/usr/src/sys/modules/mlx5en/../../ofed/include -I/usr/src/sys/modules/mlx5en/../../compat/linuxkpi/common/include -DHAVE_KERNEL_OPTION_HEADERS -include /usr/obj/usr/src/sys/ELECTRO_BLOAT/opt_global.h -I. -I/usr/src/sys -fno-common -g -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -I/usr/obj/usr/src/sys/ELECTRO_BLOAT -mcmodel=kernel -mno-red-zone -mno-mmx -mno-sse -msoft-float -fno-asynchronous-unwind-tables -ffreestanding -fwrapv -fstack-protector -gdwarf-2 -Wall -Wredundant-decls -Wnested-externs -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith -Winline -Wcast-qual -Wundef -Wno-pointer-sign -D__printf__=__freebsd_kprintf__ -Wmissing-include-dirs -fdiagnostics-show-option -Wno-unknown-pragmas -Wno-error-tautological-compare -Wno-error-empty-body -Wno-error-parentheses-equality -Wno-error-unused-function -Wno-error-pointer-sign -Wno-error-shift-negative-value -mno-aes -mno-avx -std=iso9899:1999 -Wno-cast-qual -Wno-pointer-arith -c /usr/src/sys/modules/mlx5en/../../dev/mlx5/mlx5_en/mlx5_en_ethtool.c -o mlx5_en_ethtool.o In file included from /usr/src/sys/modules/mlx5en/../../dev/mlx5/mlx5_en/mlx5_en_ethtool.c:28: In file included from /usr/src/sys/modules/mlx5en/../../dev/mlx5/mlx5_en/en.h:66: In file included from /usr/src/sys/dev/mlx5/driver.h:41: /usr/src/sys/dev/mlx5/device.h:32:10: fatal error: 'rdma/ib_verbs.h' file not found ^ 1 error generated. *** Error code 1 Stop. make[4]: stopped in /usr/src/sys/modules/mlx5en *** Error code 1 Stop. make[3]: stopped in /usr/src/sys/modules *** Error code 1 Stop. make[2]: stopped in /usr/obj/usr/src/sys/ELECTRO_BLOAT *** Error code 1 Stop. make[1]: stopped in /usr/src *** Error code 1 Stop. make: stopped in /usr/src Obtained from: ElectroBSD --- sys/modules/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 8a4f794c6132..dda252f4e7ec 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -640,11 +640,11 @@ _mlx4= mlx4 _mlx4ib= mlx4ib _mlxen= mlxen _mlx5= mlx5 -.endif .if (${MK_INET_SUPPORT} != "no" && ${MK_INET6_SUPPORT} != "no") || \ defined(ALL_MODULES) _mlx5en= mlx5en .endif +.endif _mly= mly .if ${MK_OFED} != "no" || defined(ALL_MODULES) _mthca= mthca -- 2.11.0 From 8e6beb7bf439873163b8eb9681c019cf9a0b81ce Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 29 Feb 2016 09:12:05 +0100 Subject: [PATCH 182/257] release/scripts/strip-freebsd.sh: Ditch ncsw which was added in r296177/8bf1194fe5 Obtained from: ElectroBSD --- release/scripts/strip-freebsd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh index fea0889485a6..beb9128af517 100755 --- a/release/scripts/strip-freebsd.sh +++ b/release/scripts/strip-freebsd.sh @@ -122,7 +122,7 @@ get_directories_to_ditch() { fi done - for sys_contrib in ipfilter octeon-sdk; do + for sys_contrib in ipfilter ncsw octeon-sdk; do echo "sys/contrib/${sys_contrib}" done -- 2.11.0 From 4d6f4e6fa210af1e3a77cace5c8b2963e49829a1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Mar 2016 16:54:42 +0100 Subject: [PATCH 183/257] sys/netinet: Default to not processing IP options Obtained from: ElectroBSD --- sys/netinet/ip_options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/netinet/ip_options.c b/sys/netinet/ip_options.c index 3e44ffb7b7c9..cc640d1e16bf 100644 --- a/sys/netinet/ip_options.c +++ b/sys/netinet/ip_options.c @@ -78,7 +78,7 @@ SYSCTL_INT(_net_inet_ip, IPCTL_ACCEPTSOURCEROUTE, accept_sourceroute, "Enable accepting source routed IP packets"); #define V_ip_acceptsourceroute VNET(ip_acceptsourceroute) -VNET_DEFINE(int, ip_doopts) = 1; /* 0 = ignore, 1 = process, 2 = reject */ +VNET_DEFINE(int, ip_doopts) = 0; /* 0 = ignore, 1 = process, 2 = reject */ SYSCTL_INT(_net_inet_ip, OID_AUTO, process_options, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip_doopts), 0, "Enable IP options processing ([LS]SRR, RR, TS)"); -- 2.11.0 From 899bc08ad8a1ddbee4f0a89c9db70340b7f90540 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 25 Mar 2016 23:25:33 +0100 Subject: [PATCH 184/257] Detach freebsd-version from the build Obtained from: ElectroBSD --- bin/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/Makefile b/bin/Makefile index b3385dcd32d9..eb89a529abbe 100644 --- a/bin/Makefile +++ b/bin/Makefile @@ -15,7 +15,6 @@ SUBDIR= cat \ echo \ ed \ expr \ - freebsd-version \ getfacl \ hostname \ kenv \ -- 2.11.0 From 34ecd7a8e1f3a0485cc9f1cfd9cdf0bcd92a1043 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 25 Mar 2016 18:03:20 +0100 Subject: [PATCH 185/257] geli(8): Dump negative iterations values properly An iterations value of -1 signals that no passphrase is being used. Previously it was dumped incorrectly due to being cast to unsigned first: [fk@elektrobier ~]$ geli dump /dev/ada3p3 Metadata on /dev/ada3p3: magic: GEOM::ELI version: 7 flags: 0x2 ealgo: AES-XTS keylen: 256 provsize: 8589934592 sectorsize: 4096 keys: 0x01 iterations: 4294967295 Salt: [...] Master Key: [...] MD5 hash: 44e17ea9c25f27dff32179bc4f82f659 Obtained from: ElectroBSD --- sys/geom/eli/g_eli.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h index 13e780762549..680f67332dd3 100644 --- a/sys/geom/eli/g_eli.h +++ b/sys/geom/eli/g_eli.h @@ -499,7 +499,7 @@ eli_metadata_dump(const struct g_eli_metadata *md) printf(" provsize: %ju\n", (uintmax_t)md->md_provsize); printf("sectorsize: %u\n", (u_int)md->md_sectorsize); printf(" keys: 0x%02x\n", (u_int)md->md_keys); - printf("iterations: %u\n", (u_int)md->md_iterations); + printf("iterations: %i\n", md->md_iterations); bzero(str, sizeof(str)); for (i = 0; i < sizeof(md->md_salt); i++) { str[i * 2] = hex[md->md_salt[i] >> 4]; -- 2.11.0 From 6a5c94e88d63b07836190057aeb6d0f7ae719abb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Mar 2016 15:14:25 +0200 Subject: [PATCH 186/257] geli.8: Document that setkey option -P can't be combined with -i Obtained from: ElectroBSD --- sbin/geom/class/eli/geli.8 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbin/geom/class/eli/geli.8 b/sbin/geom/class/eli/geli.8 index e8a1ef024c1b..3c5111928c9f 100644 --- a/sbin/geom/class/eli/geli.8 +++ b/sbin/geom/class/eli/geli.8 @@ -554,6 +554,8 @@ option. .It Fl P Do not use a passphrase as a component of the new User Key. Cannot be combined with the +.Fl i +or .Fl J option. .El -- 2.11.0 From 09f75af731534bd6482cff4d6acc0a32d1a56cc6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Mar 2016 15:19:53 +0200 Subject: [PATCH 187/257] geli setkey: Reject combination of -P and -i Iterations aren't used for keys without passphrase and storing a iteration count would result in a passphrase promt that can't be used to attach the provider. Obtained from: ElectroBSD --- sbin/geom/class/eli/geom_eli.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sbin/geom/class/eli/geom_eli.c b/sbin/geom/class/eli/geom_eli.c index c804622ddbe2..e8198d118179 100644 --- a/sbin/geom/class/eli/geom_eli.c +++ b/sbin/geom/class/eli/geom_eli.c @@ -1166,6 +1166,8 @@ eli_setkey(struct gctl_req *req) struct g_eli_metadata md; const char *prov; int nargs; + intmax_t iterations; + int nonewpassphrase; nargs = gctl_get_int(req, "nargs"); if (nargs != 1) { @@ -1174,6 +1176,14 @@ eli_setkey(struct gctl_req *req) } prov = gctl_get_ascii(req, "arg0"); + nonewpassphrase = gctl_get_int(req, "nonewpassphrase"); + iterations = gctl_get_intmax(req, "iterations"); + if (iterations != -1 && nonewpassphrase) { + gctl_error(req, + "Options -i and -P are mutually exclusive."); + return; + } + if (eli_metadata_read(req, prov, &md) == -1) return; -- 2.11.0 From ddb1d411be5834d239538f8f40fc5156de2386f5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Mar 2016 15:24:02 +0200 Subject: [PATCH 188/257] geli setkey: Prevent passphrase removal if two keys are configured The iterations are shared by both keys and resetting it for one would break the other one as a side-effect. If this is the intention the other key can still be removed with delkey before using setkey for the remaining one. Obtained from: ElectroBSD --- sbin/geom/class/eli/geom_eli.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sbin/geom/class/eli/geom_eli.c b/sbin/geom/class/eli/geom_eli.c index e8198d118179..bd79b34ff1f6 100644 --- a/sbin/geom/class/eli/geom_eli.c +++ b/sbin/geom/class/eli/geom_eli.c @@ -1187,6 +1187,14 @@ eli_setkey(struct gctl_req *req) if (eli_metadata_read(req, prov, &md) == -1) return; + if (nonewpassphrase && bitcount32(md.md_keys) != 1 && + md.md_iterations != -1) { + gctl_error(req, "To be able to switch from passphrase" + "-based key to passphrase-less key, only one can " + "be defined."); + return; + } + if (eli_is_attached(prov)) eli_setkey_attached(req, &md); else -- 2.11.0 From a763ebd8e38c2bc8ffc65640ab5101bb317cb8c9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Mar 2016 15:37:03 +0200 Subject: [PATCH 189/257] geli.8: Document that setkey -P can't be used if two user keys with passphrase exist Obtained from: ElectroBSD --- sbin/geom/class/eli/geli.8 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/geom/class/eli/geli.8 b/sbin/geom/class/eli/geli.8 index 3c5111928c9f..22660544db53 100644 --- a/sbin/geom/class/eli/geli.8 +++ b/sbin/geom/class/eli/geli.8 @@ -553,6 +553,9 @@ Cannot be combined with the option. .It Fl P Do not use a passphrase as a component of the new User Key. +This option cannot be used with the +.Cm setkey +subcommand if two User Keys with passphrase are configured. Cannot be combined with the .Fl i or -- 2.11.0 From 22e26658a8b23897a2c90b7a0e695562948e2510 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 27 Mar 2016 14:22:52 +0200 Subject: [PATCH 190/257] geli setkey: Fix key change from passphrase-based to passphrase-less Previously the iterations count wasn't reset resulting in a passphrase prompt that would not accept any passphrase: fk@r500 ~ $sudo geli attach -k /bpool/boot/rpool.key /dev/gpt/rpool-ada1 Enter passphrase: fk@r500 ~ $geli dump /dev/gpt/rpool-ada1 Metadata on /dev/gpt/rpool-ada1: magic: GEOM::ELI version: 7 flags: 0x0 ealgo: AES-XTS keylen: 128 provsize: 1073741824 sectorsize: 512 keys: 0x01 iterations: 463852 Salt: [...] Master Key: [...] MD5 hash: e8693274fecc65d2a13c0071fb9413b3 fk@r500 ~ $sudo geli setkey -K /bpool/boot/rpool.key -P /dev/gpt/rpool-ada1 Note, that the master key encrypted with old keys and/or passphrase may still exists in a metadata backup file. fk@r500 ~ $geli dump /dev/gpt/rpool-ada1 Metadata on /dev/gpt/rpool-ada1: magic: GEOM::ELI version: 7 flags: 0x0 ealgo: AES-XTS keylen: 128 provsize: 1073741824 sectorsize: 512 keys: 0x01 iterations: 463852 Salt: [...] Master Key: [...] MD5 hash: a443402c3b97cb37494283f8f722994d fk@r500 ~ $sudo geli detach gpt/rpool-ada1 fk@r500 ~ $sudo geli attach -k /bpool/boot/rpool.key /dev/gpt/rpool-ada1 Enter passphrase: geli: Wrong key for gpt/rpool-ada1. From userland the promt could be suppressed with "-p" (which is required for passphrase-less keys anyway), but attaching at boot time wasn't possible. PR: 196834 Reported by: Julian Hsiao Obtained from: ElectroBSD --- sbin/geom/class/eli/geom_eli.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/sbin/geom/class/eli/geom_eli.c b/sbin/geom/class/eli/geom_eli.c index bd79b34ff1f6..856a862fe0fe 100644 --- a/sbin/geom/class/eli/geom_eli.c +++ b/sbin/geom/class/eli/geom_eli.c @@ -1044,8 +1044,16 @@ eli_setkey_attached(struct gctl_req *req, struct g_eli_metadata *md) /* Check if iterations number should be changed. */ if (val != -1) md->md_iterations = val; - else + else { old = md->md_iterations; + /* + * If the new key does not require a passphrase, + * the iterations count has to be reset to reflect + * this. + */ + if (gctl_get_int(req, "nonewpassphrase") == 1) + md->md_iterations = -1; + } /* Generate key for Master Key encryption. */ if (eli_genkey(req, md, key, true) == NULL) { @@ -1130,7 +1138,14 @@ eli_setkey_detached(struct gctl_req *req, const char *prov, return; } md->md_iterations = val; - } + } else if (gctl_get_int(req, "nonewpassphrase") == 1) { + /* + * If the new key does not require a passphrase, + * the iterations count has to be reset to reflect + * this. + */ + md->md_iterations = -1; + } mkeydst = md->md_mkeys + nkey * G_ELI_MKEYLEN; md->md_keys |= (1 << nkey); -- 2.11.0 From f01ffc62995008321006fb5f613c0c15acac3dfe Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 4 Apr 2016 17:37:05 +0200 Subject: [PATCH 191/257] geli.8: Document the fact that both User Keys share an iteration value ... more explicitly. Obtained from: ElectroBSD --- sbin/geom/class/eli/geli.8 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sbin/geom/class/eli/geli.8 b/sbin/geom/class/eli/geli.8 index 22660544db53..fdbf60c84240 100644 --- a/sbin/geom/class/eli/geli.8 +++ b/sbin/geom/class/eli/geli.8 @@ -770,6 +770,13 @@ the kernel from the Master Key and cached in memory. The number of Data Keys used by a given provider, and the way they are derived, depend on the GELI version and whether the provider is configured to use data authentication. +.Sh USER KEY LIMITATION +If the first User Key uses a passphrase, the second one has to use +a passphrase as well. +If the first User Key does not use a passphrase, the second one can +not use a passphrase either. +This limitation comes from the metadata format on disk which +currently only stores one iteration count for both keys. .Sh SYSCTL VARIABLES The following .Xr sysctl 8 -- 2.11.0 From 185af5a60ca7fab3612d32f05d0e0c97776953c2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 Apr 2016 11:45:44 +0200 Subject: [PATCH 192/257] etc/rc.d: Remove a couple of requirements that don't apply to ElectroBSD Obtained from: ElectroBSD --- etc/rc.d/NETWORKING | 4 ++-- etc/rc.d/bridge | 2 +- etc/rc.d/ipfw | 1 - etc/rc.d/ipnat | 29 ----------------------------- etc/rc.d/netif | 4 ++-- etc/rc.d/routing | 2 +- etc/rc.d/securelevel | 2 +- 7 files changed, 7 insertions(+), 37 deletions(-) delete mode 100755 etc/rc.d/ipnat diff --git a/etc/rc.d/NETWORKING b/etc/rc.d/NETWORKING index 9cdb5577ed2b..1bbea5aa40b9 100755 --- a/etc/rc.d/NETWORKING +++ b/etc/rc.d/NETWORKING @@ -4,8 +4,8 @@ # # PROVIDE: NETWORKING NETWORK -# REQUIRE: netif netwait netoptions routing ppp ipfw stf -# REQUIRE: defaultroute route6d resolv bridge +# REQUIRE: netif netwait netoptions routing ipfw stf +# REQUIRE: defaultroute routed route6d resolv bridge # REQUIRE: static_arp static_ndp # This is a dummy dependency, for services which require networking diff --git a/etc/rc.d/bridge b/etc/rc.d/bridge index 95e4eb9c2fac..8631fdab2b05 100755 --- a/etc/rc.d/bridge +++ b/etc/rc.d/bridge @@ -26,7 +26,7 @@ # # PROVIDE: bridge -# REQUIRE: netif ppp stf +# REQUIRE: netif stf # KEYWORD: nojail . /etc/rc.subr diff --git a/etc/rc.d/ipfw b/etc/rc.d/ipfw index beccaffaf31b..7f9db1656de2 100755 --- a/etc/rc.d/ipfw +++ b/etc/rc.d/ipfw @@ -4,7 +4,6 @@ # # PROVIDE: ipfw -# REQUIRE: ppp # KEYWORD: nojailvnet . /etc/rc.subr diff --git a/etc/rc.d/ipnat b/etc/rc.d/ipnat deleted file mode 100755 index 6914bceaf466..000000000000 --- a/etc/rc.d/ipnat +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# -# $FreeBSD$ -# - -# PROVIDE: ipnat -# REQUIRE: ipfilter -# KEYWORD: nojail - -. /etc/rc.subr - -name="ipnat" -desc="user interface to the NAT subsystem" -rcvar="ipnat_enable" -load_rc_config $name -start_cmd="ipnat_start" -stop_cmd="${ipnat_program} -F -C" -reload_cmd="${ipnat_program} -F -C -f ${ipnat_rules}" -extra_commands="reload" -required_files="${ipnat_rules}" -required_modules="ipl:ipfilter" - -ipnat_start() -{ - echo "Installing NAT rules." - ${ipnat_program} -CF -f ${ipnat_rules} ${ipnat_flags} -} - -run_rc_command "$1" diff --git a/etc/rc.d/netif b/etc/rc.d/netif index 6b52c50e7f4f..2458f0a863f5 100755 --- a/etc/rc.d/netif +++ b/etc/rc.d/netif @@ -26,8 +26,8 @@ # # PROVIDE: netif -# REQUIRE: FILESYSTEMS iovctl serial sppp sysctl -# REQUIRE: hostid ipfilter ipfs +# REQUIRE: FILESYSTEMS iovctl serial sysctl +# REQUIRE: hostid # KEYWORD: nojailvnet . /etc/rc.subr diff --git a/etc/rc.d/routing b/etc/rc.d/routing index 2a57fa3b1182..7de71f12c9e1 100755 --- a/etc/rc.d/routing +++ b/etc/rc.d/routing @@ -6,7 +6,7 @@ # # PROVIDE: routing -# REQUIRE: netif ppp stf +# REQUIRE: netif stf # KEYWORD: nojailvnet . /etc/rc.subr diff --git a/etc/rc.d/securelevel b/etc/rc.d/securelevel index c42a03534675..24dbf269df3f 100755 --- a/etc/rc.d/securelevel +++ b/etc/rc.d/securelevel @@ -4,7 +4,7 @@ # # PROVIDE: securelevel -# REQUIRE: adjkerntz ipfw ipfilter pf +# REQUIRE: adjkerntz ipfw pf . /etc/rc.subr -- 2.11.0 From fdc6e512b1b870dd773dd91ff6ef88bf22dc7f6e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 16 Apr 2016 18:17:16 +0200 Subject: [PATCH 193/257] etc/rc.d: Don't try to install ipnat Obtained from: ElectroBSD --- etc/rc.d/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile index ac3dda4100a2..213e90c4c8d1 100644 --- a/etc/rc.d/Makefile +++ b/etc/rc.d/Makefile @@ -46,7 +46,6 @@ FILES= DAEMON \ ipfs \ ipfw \ ipmon \ - ipnat \ ipsec \ ${_kadmind} \ ${_kdc} \ -- 2.11.0 From 401565ae8c294a9c874c311dc0fc0f068e2a1feb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 20 Apr 2016 21:45:39 +0200 Subject: [PATCH 194/257] release/amd64/make-memstick.sh: Use mkimg from the dist directory Obtained from: ElectroBSD --- release/amd64/make-memstick.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 4263855e011d..3925607dfcd3 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -50,7 +50,8 @@ rm ${1}/etc/fstab rm ${1}/etc/rc.conf.local rm "${2}.mtree" -mkimg -y -s gpt -b ${1}/boot/pmbr -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} +dist/base/usr/bin/mkimg -y -s gpt -b ${1}/boot/pmbr \ + -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} if [ $? -ne 0 ]; then echo "mkimg failed" exit 1 -- 2.11.0 From ecc0b7a9c7de82dd1dc297169698e70c7b51bb41 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 18 Mar 2015 13:08:28 +0100 Subject: [PATCH 195/257] parse_mount(): Use vfs.mountroot.timeout for ZFS root pools as well Instead of trying to figure out the required vdevs to wait for, just call kernel_mount() until it works or the time is up. Fancier approaches are conceivable ... If the current approach is kept, it may make sense to add a flag to tell kernel_mount() not to free the mount args. As an alternative the retrying could be delegated to kernel_mount() itself. Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=208882 --- sys/kern/vfs_mountroot.c | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c index 248f6c887c64..ff437c3f4008 100644 --- a/sys/kern/vfs_mountroot.c +++ b/sys/kern/vfs_mountroot.c @@ -707,7 +707,7 @@ parse_mount(char **conf) char *errmsg; struct mntarg *ma; char *dev, *fs, *opts, *tok; - int error; + int delay, error, timeout; error = parse_token(conf, &tok); if (error) @@ -748,15 +748,29 @@ parse_mount(char **conf) if (error != 0) goto out; - ma = NULL; - ma = mount_arg(ma, "fstype", fs, -1); - ma = mount_arg(ma, "fspath", "/", -1); - ma = mount_arg(ma, "from", dev, -1); - ma = mount_arg(ma, "errmsg", errmsg, ERRMSGL); - ma = mount_arg(ma, "ro", NULL, 0); - ma = parse_mountroot_options(ma, opts); - error = kernel_mount(ma, MNT_ROOTFS); + delay = hz / 10; + timeout = root_mount_timeout * hz; + do { + ma = NULL; + ma = mount_arg(ma, "fstype", fs, -1); + ma = mount_arg(ma, "fspath", "/", -1); + ma = mount_arg(ma, "from", dev, -1); + ma = mount_arg(ma, "errmsg", errmsg, ERRMSGL); + ma = mount_arg(ma, "ro", NULL, 0); + ma = parse_mountroot_options(ma, opts); + + error = kernel_mount(ma, MNT_ROOTFS); + if (strcmp(fs, "zfs") != 0) + break; + timeout -= delay; + if (timeout > 0 && error) { + pause("rmdev", delay); + printf("Mounting from %s:%s failed with error %d. " + "%d seconds left. Retrying.\n", fs, dev, error, + timeout / hz); + } + } while (timeout > 0 && error); out: if (error) { printf("Mounting from %s:%s failed with error %d", @@ -960,8 +974,12 @@ vfs_mountroot_wait_if_neccessary(const char *fs, const char *dev) int delay, timeout; /* - * In case of ZFS and NFS we don't have a way to wait for - * specific device. + * For ZFS we can't simply wait for a specific device + * as we only know the pool name. To work around this, + * parse_mount() will retry the mount later on. + * + * While retrying for NFS could be implemented similarly + * it is currently not supported. */ if (strcmp(fs, "zfs") == 0 || strstr(fs, "nfs") != NULL || dev[0] == '\0') { -- 2.11.0 From aa3fcc004e73830a157bfe540e377639014f329e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 1 Nov 2015 14:21:15 +0100 Subject: [PATCH 196/257] parse_mount(): Use the vfs.mountroot.timeout for NFS as well Note that this hasn't actually been tested yet. Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=208882 --- sys/kern/vfs_mountroot.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c index ff437c3f4008..5698ba03d30d 100644 --- a/sys/kern/vfs_mountroot.c +++ b/sys/kern/vfs_mountroot.c @@ -761,7 +761,7 @@ parse_mount(char **conf) ma = parse_mountroot_options(ma, opts); error = kernel_mount(ma, MNT_ROOTFS); - if (strcmp(fs, "zfs") != 0) + if (strcmp(fs, "zfs") != 0 || strstr(fs, "nfs") != NULL) break; timeout -= delay; if (timeout > 0 && error) { @@ -974,12 +974,9 @@ vfs_mountroot_wait_if_neccessary(const char *fs, const char *dev) int delay, timeout; /* - * For ZFS we can't simply wait for a specific device + * For ZFS and NFS we can't simply wait for a specific device * as we only know the pool name. To work around this, * parse_mount() will retry the mount later on. - * - * While retrying for NFS could be implemented similarly - * it is currently not supported. */ if (strcmp(fs, "zfs") == 0 || strstr(fs, "nfs") != NULL || dev[0] == '\0') { -- 2.11.0 From 34efdd2af8feebbad8fd0aaa9aca459e5f84d98f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 Apr 2016 12:48:33 +0200 Subject: [PATCH 197/257] rc.d: Change geli_autodetach default to 'NO' ... as autodetach leads to panics when scrubbing ZFS pools with more than one device. For details see: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=117158 Obtained from: ElectroBSD --- etc/defaults/rc.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 etc/defaults/rc.conf diff --git a/etc/defaults/rc.conf b/etc/defaults/rc.conf old mode 100644 new mode 100755 index 18ee897bfb58..c1494f933edf --- a/etc/defaults/rc.conf +++ b/etc/defaults/rc.conf @@ -78,7 +78,7 @@ geli_devices="" # List of devices to automatically attach in addition to geli_tries="" # Number of times to attempt attaching geli device. # If empty, kern.geom.eli.tries will be used. geli_default_flags="" # Default flags for geli(8). -geli_autodetach="YES" # Automatically detach on last close. +geli_autodetach="NO" # Automatically detach on last close. # Providers are marked as such when all file systems are # mounted. # Example use. -- 2.11.0 From a1c8f58bb2a7793749446588918a044b32bd10ee Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 May 2016 22:05:48 +0200 Subject: [PATCH 198/257] disklatency: Skip invalid io::done probes .. until the cause has been analyzed and fixed Obtained from: ElectroBSD --- share/dtrace/disklatency | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/dtrace/disklatency b/share/dtrace/disklatency index 97ef87a07980..21ff6f719053 100755 --- a/share/dtrace/disklatency +++ b/share/dtrace/disklatency @@ -42,7 +42,7 @@ io:::start } io:::done -/this->start = start_time[arg0]/ +/(this->start = start_time[arg0]) && (args[1]->unit_number != -1)/ { this->delta = (timestamp - this->start) / 1000; @q[args[1]->device_name, args[1]->unit_number] = -- 2.11.0 From c35664de36440370002e79fdfc31c6b40170b84e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 May 2016 22:06:51 +0200 Subject: [PATCH 199/257] disklatency: Reformat output to print device name and unit together While at it, add a delimiter. Obtained from: ElectroBSD --- share/dtrace/disklatency | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/share/dtrace/disklatency b/share/dtrace/disklatency index 21ff6f719053..930be21d0afb 100755 --- a/share/dtrace/disklatency +++ b/share/dtrace/disklatency @@ -55,10 +55,11 @@ io:::done tick-10s { - printa(" %s (%d), us:\n%@d\n", @q); - printa("max%s (%d), us:\n%@d\n", @max); - printa("avg%s (%d), us:\n%@d\n", @avg); - printa("stddev%s (%d), us:\n%@d\n", @stddev); + printf("---------------------------------------------\n"); + printa("%s%d, us:\n%@d\n", @q); + printa("%s%d max, us:\n%@d\n", @max); + printa("%s%d avg, us:\n%@d\n", @avg); + printa("%s%d stdev, us:\n%@d\n", @stddev); clear(@q); clear(@max); -- 2.11.0 From e462065a6f38c19fc384d80106fa6fca3708a2f0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 15 May 2016 09:30:31 +0200 Subject: [PATCH 200/257] disklatency: Use trunc() instead of clear() ... so only latency stats for disks with activity in the relevant intervals are shown. Obtained from: ElectroBSD --- share/dtrace/disklatency | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/share/dtrace/disklatency b/share/dtrace/disklatency index 930be21d0afb..f4081c14bc84 100755 --- a/share/dtrace/disklatency +++ b/share/dtrace/disklatency @@ -61,8 +61,8 @@ tick-10s printa("%s%d avg, us:\n%@d\n", @avg); printa("%s%d stdev, us:\n%@d\n", @stddev); - clear(@q); - clear(@max); - clear(@avg); - clear(@stddev); + trunc(@q); + trunc(@max); + trunc(@avg); + trunc(@stddev); } -- 2.11.0 From 29bd43fd4d8d2ca15e11bd5f87ce4fb64fed46dc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 1 May 2016 16:27:55 +0200 Subject: [PATCH 201/257] vmstat: Prevent output truncation when piping zone statistics ... into other programs. Apparently libxo (currently?) can't be trusted to flush its internal buffer before it's full, triggering the flushing manually works around this. Before (sometimes): fk@r500 ~ $vmstat -z | wc -c 8192 After (consistently): fk@r500 ~ $vmstat -z | wc -c 24156 Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=206128 --- usr.bin/vmstat/vmstat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/usr.bin/vmstat/vmstat.c b/usr.bin/vmstat/vmstat.c index 0dbd2d165dda..a4d6f25786c1 100644 --- a/usr.bin/vmstat/vmstat.c +++ b/usr.bin/vmstat/vmstat.c @@ -1536,6 +1536,7 @@ domemstat_zone(void) memstat_get_numallocs(mtp), memstat_get_failures(mtp), memstat_get_sleeps(mtp)); xo_close_instance("zone"); + xo_flush(); } memstat_mtl_free(mtlp); xo_close_list("zone"); -- 2.11.0 From 52c60165e1ab455594148867b06351d38e43ecf0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 23 May 2016 14:48:21 +0200 Subject: [PATCH 202/257] gptzfsboot: Print Elektrobier after failing to locate a ZFS pool Obtained from: ElectroBSD --- sys/boot/i386/zfsboot/zfsboot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/i386/zfsboot/zfsboot.c b/sys/boot/i386/zfsboot/zfsboot.c index 399851152a2b..62332d83cd4c 100644 --- a/sys/boot/i386/zfsboot/zfsboot.c +++ b/sys/boot/i386/zfsboot/zfsboot.c @@ -706,7 +706,7 @@ main(void) if (!spa) { printf("%s: No ZFS pools located, can't boot\n", BOOTPROG); for (;;) - ; + printf("Elektrobier, "); } primary_spa = spa; -- 2.11.0 From 32a6cadabcc99e27ede924a47bb4f8aa92129b0c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 23 May 2016 16:02:08 +0200 Subject: [PATCH 203/257] uma: Add UMA_ZONE_NODRAIN flag INCOMPLETE and non-functional Obtained from: ElectroBSD --- sys/vm/uma.h | 2 +- sys/vm/uma_core.c | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sys/vm/uma.h b/sys/vm/uma.h index f4c2de8a5d25..fcdb7449f830 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -262,7 +262,7 @@ uma_zone_t uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor, * information in the vm_page. */ #define UMA_ZONE_SECONDARY 0x0200 /* Zone is a Secondary Zone */ -/* 0x0400 Unused */ +#define UMA_ZONE_NODRAIN 0x0400 /* Do not drain zone while in use */ #define UMA_ZONE_MAXBUCKET 0x0800 /* Use largest buckets */ #define UMA_ZONE_CACHESPREAD 0x1000 /* * Spread memory start locations across diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index e54db2a29740..560851bb10e8 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -855,6 +855,9 @@ keg_drain(uma_keg_t keg) if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) return; + if (keg->uk_flags & UMA_ZONE_NODRAIN) + return; + #ifdef UMA_DEBUG printf("%s free items: %u\n", keg->uk_name, keg->uk_free); #endif @@ -2077,6 +2080,7 @@ uma_zdestroy(uma_zone_t zone) { sx_slock(&uma_drain_lock); + zone->uz_flags &= ~UMA_ZONE_NODRAIN; zone_free_item(zones, zone, NULL, SKIP_NONE); sx_sunlock(&uma_drain_lock); } -- 2.11.0 From 0e2bf441acb20f77803e7ffd08dbb6504aa0d175 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 23 May 2016 13:06:34 +0200 Subject: [PATCH 204/257] zone.9: Stop documenting UMA_ZONE_REFCNT which was removed in r296243 Obtained from: ElectroBSD --- share/man/man9/zone.9 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/share/man/man9/zone.9 b/share/man/man9/zone.9 index 69d1f20b498e..798be1c20fdf 100644 --- a/share/man/man9/zone.9 +++ b/share/man/man9/zone.9 @@ -155,10 +155,6 @@ is a subset of the following flags: .Bl -tag -width "foo" .It Dv UMA_ZONE_NOFREE Slabs of the zone are never returned back to VM. -.It Dv UMA_ZONE_REFCNT -Each item in the zone would have internal reference counter associated with it. -See -.Fn uma_find_refcnt . .It Dv UMA_ZONE_NODUMP Pages belonging to the zone will not be included into mini-dumps. .It Dv UMA_ZONE_PCPU -- 2.11.0 From 3494957d9c27d027bf6379a9a31de57e1468f09f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 May 2016 16:53:50 +0200 Subject: [PATCH 205/257] uma.h: Rename uma_prealloc()'s second argument to nitems to increase consistency Obtained from: ElectroBSD --- sys/vm/uma.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/vm/uma.h b/sys/vm/uma.h index fcdb7449f830..b1fc43f713cf 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -613,14 +613,14 @@ void uma_zone_set_freef(uma_zone_t zone, uma_free freef); * * Arguments: * zone The zone to fill - * itemcnt The number of items to reserve + * nitems The number of items to reserve * * Returns: * Nothing * * NOTE: This is blocking and should only be done at startup */ -void uma_prealloc(uma_zone_t zone, int itemcnt); +void uma_prealloc(uma_zone_t zone, int nitems); /* * Used to determine if a fixed-size zone is exhausted. -- 2.11.0 From 3b9283c589c1a0092d017c7f4c8ba24fddb2dffb Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 May 2016 17:25:00 +0200 Subject: [PATCH 206/257] zone.9: Document uma_prealloc() and uma_zone_reserve() Obtained from: ElectroBSD --- share/man/man9/zone.9 | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/share/man/man9/zone.9 b/share/man/man9/zone.9 index 798be1c20fdf..3636a5ecfa98 100644 --- a/share/man/man9/zone.9 +++ b/share/man/man9/zone.9 @@ -29,6 +29,7 @@ .Dt ZONE 9 .Os .Sh NAME +.Nm uma_prealloc , .Nm uma_zcreate , .Nm uma_zalloc , .Nm uma_zalloc_arg , @@ -36,6 +37,7 @@ .Nm uma_zfree_arg , .Nm uma_find_refcnt , .Nm uma_zdestroy , +.Nm uma_zone_reserve , .Nm uma_zone_set_max, .Nm uma_zone_get_max, .Nm uma_zone_get_cur, @@ -46,6 +48,8 @@ .In sys/param.h .In sys/queue.h .In vm/uma.h +.Ft void +.Fn uma_prealloc "uma_zone_t zone" "int nitems" .Ft uma_zone_t .Fo uma_zcreate .Fa "char *name" "int size" @@ -64,6 +68,8 @@ .Fn uma_find_refcnt "uma_zone_t zone" "void *item" .Ft void .Fn uma_zdestroy "uma_zone_t zone" +.Ft void +.Fn uma_zone_reserve "uma_zone_t zone" "int nitems" .Ft int .Fn uma_zone_set_max "uma_zone_t zone" "int nitems" .Ft int @@ -272,6 +278,21 @@ must have been freed with before. .Pp The +.Fn uma_prealloc +function pre-fills a zone with items. +This function is blocking and should be called before the zone is used. +.Pp +The +.Fn uma_zone_reserve +function sets a reserve of items to hold for +.Dv M_USE_RESERVE +allocations. +No new items are allocated. +The +.Fn uma_prealloc +function can be used to fill the reserve before it is needed. +.Pp +The .Fn uma_zone_set_max function limits the number of items .Pq and therefore memory -- 2.11.0 From b70f80640c5c75d01380a565cc8c6a8620ed9169 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 16 May 2016 12:28:50 +0200 Subject: [PATCH 207/257] uma.h: Stop claiming that uma_zcreate() may return NULL ... 'if the wait flag is not set'. The function does not actually accept a "wait flag". Internally it unconditionally uses the M_WAITOK flag when calling zone_alloc_item(). Obtained from: ElectroBSD --- sys/vm/uma.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/vm/uma.h b/sys/vm/uma.h index b1fc43f713cf..ce20edc82004 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -173,7 +173,7 @@ typedef void (*uma_release)(void *arg, void **store, int count); * * Returns: * A pointer to a structure which is intended to be opaque to users of - * the interface. The value may be null if the wait flag is not set. + * the interface. */ uma_zone_t uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, uma_init uminit, uma_fini fini, -- 2.11.0 From 4c103fdad9f95e4c3bf09e438ffc678209383f35 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 22 May 2016 12:47:29 +0200 Subject: [PATCH 208/257] ipfw: Prefill the dynamic rule zone and prevent uma from freeing unused items Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=209680 --- sys/netpfil/ipfw/ip_fw_dynamic.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sys/netpfil/ipfw/ip_fw_dynamic.c b/sys/netpfil/ipfw/ip_fw_dynamic.c index 7dd72e47dd9e..f1fda48d28a6 100644 --- a/sys/netpfil/ipfw/ip_fw_dynamic.c +++ b/sys/netpfil/ipfw/ip_fw_dynamic.c @@ -1399,11 +1399,14 @@ ipfw_dyn_init(struct ip_fw_chain *chain) V_ipfw_dyn_rule_zone = uma_zcreate("IPFW dynamic rule", sizeof(ipfw_dyn_rule), NULL, NULL, NULL, NULL, - UMA_ALIGN_PTR, 0); + UMA_ALIGN_PTR, UMA_ZONE_NOFREE); /* Enforce limit on dynamic rules */ uma_zone_set_max(V_ipfw_dyn_rule_zone, V_dyn_max); + /* Prefill the zone to make sure the limit is always reachable */ + uma_prealloc(V_ipfw_dyn_rule_zone, V_dyn_max); + callout_init(&V_ipfw_timeout, 1); /* -- 2.11.0 From fcd1e315c7409916c49a7932cd1e5b88f60e40b4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 14 May 2016 13:15:31 +0200 Subject: [PATCH 209/257] (Hopefully) make TCP/IP connections reliable under memory pressure again ... by bringing back the UMA_ZONE_NOFREE flag for a couple of TCP related UMA zones. This reverts r297740/f3240321 completely and partially reverts r297738/55f63f4b6bf and r297742/0a1a3e73fc. Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=209680 --- sys/netinet/tcp_fastopen.c | 2 +- sys/netinet/tcp_subr.c | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sys/netinet/tcp_fastopen.c b/sys/netinet/tcp_fastopen.c index e2b3b8637f60..730032a63daf 100644 --- a/sys/netinet/tcp_fastopen.c +++ b/sys/netinet/tcp_fastopen.c @@ -205,7 +205,7 @@ void tcp_fastopen_init(void) { V_counter_zone = uma_zcreate("tfo", sizeof(unsigned int), - NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); rm_init(&V_tcp_fastopen_keylock, "tfo_keylock"); callout_init_rm(&V_tcp_fastopen_autokey_ctx.c, &V_tcp_fastopen_keylock, 0); diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c index 662c435a0e72..1c45a9a6a117 100644 --- a/sys/netinet/tcp_subr.c +++ b/sys/netinet/tcp_subr.c @@ -651,13 +651,14 @@ tcp_init(void) hashsize); } in_pcbinfo_init(&V_tcbinfo, "tcp", &V_tcb, hashsize, hashsize, - "tcp_inpcb", tcp_inpcb_init, NULL, 0, IPI_HASHFIELDS_4TUPLE); + "tcp_inpcb", tcp_inpcb_init, NULL, UMA_ZONE_NOFREE, + IPI_HASHFIELDS_4TUPLE); /* * These have to be type stable for the benefit of the timers. */ V_tcpcb_zone = uma_zcreate("tcpcb", sizeof(struct tcpcb_mem), - NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); uma_zone_set_max(V_tcpcb_zone, maxsockets); uma_zone_set_warning(V_tcpcb_zone, "kern.ipc.maxsockets limit reached"); @@ -667,7 +668,7 @@ tcp_init(void) TUNABLE_INT_FETCH("net.inet.tcp.sack.enable", &V_tcp_do_sack); V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole), - NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); /* Skip initialization of globals for non-default instances. */ if (!IS_DEFAULT_VNET(curvnet)) -- 2.11.0 From 3688caa2eddaefddde4cfa66fa08853c332a84fc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 15 May 2016 12:09:26 +0200 Subject: [PATCH 210/257] Add geli-request-monitor.d ... which shows the BIO commands GELI received and their lengths from the caller's point of view. Obtained from: ElectroBSD --- share/dtrace/geli-request-monitor.d | 86 +++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100755 share/dtrace/geli-request-monitor.d diff --git a/share/dtrace/geli-request-monitor.d b/share/dtrace/geli-request-monitor.d new file mode 100755 index 000000000000..c306d15bff96 --- /dev/null +++ b/share/dtrace/geli-request-monitor.d @@ -0,0 +1,86 @@ +#!/usr/sbin/dtrace -s + +/*************************************************************************** + * geli-request-monitor + * + * Shows the length distribution of various GELI commands from + * the callers point of view (that is, without geli overhead). + * + * Copyright (c) 2016 Fabian Keil + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************/ + +#pragma D option quiet + +dtrace:::BEGIN +{ + bio_command[0x01] = "BIO_READ"; + bio_command[0x02] = "BIO_WRITE"; + bio_command[0x04] = "BIO_DELETE"; + bio_command[0x08] = "BIO_GETATTR"; + bio_command[0x10] = "BIO_FLUSH"; + bio_command[0x20] = "BIO_CMD0"; + bio_command[0x40] = "BIO_CMD1"; + bio_command[0x80] = "BIO_CMD2"; +} + + +fbt::g_eli_crypto_run:entry +{ + this->bp = (struct bio *)arg1; + this->sc = (struct g_eli_softc *)this->bp->bio_to->geom->softc; + + this->geom_name = stringof(this->sc->sc_geom->name); + this->length = this->bp->bio_length; + this->inflight = this->sc->sc_inflight; + + this->bio_command = (bio_command[this->bp->bio_cmd] != NULL) ? + bio_command[this->bp->bio_cmd] : "Unregistered command"; + + @inflight_max[this->geom_name] = max(this->inflight); + @inflight_max_total[this->geom_name] = max(this->inflight); + + @length[this->geom_name, this->bio_command] = + lquantize(this->length, 4096, 172032, 4096); + @length_total[this->geom_name, this->bio_command] = + lquantize(this->length, 4096, 172032, 4096); +/* + printf("%s: Length: %d, Inflight: %d", this->bio_command, + this->length, this->inflight); +*/ + + stats_available = 1; +} + +tick-60s +/stats_available/ +{ + printf("------------------------------\n"); + printa(@inflight_max); + printa(@length); + trunc(@length); + trunc(@inflight_max); + stats_available = 0; +} + +END +{ + /* Explicitly ditch interval stats so they aren't printed */ + trunc(@length); + trunc(@inflight_max); + + printf("\n------------------------------\n"); + printa(@inflight_max_total); + printa(@length_total); +} -- 2.11.0 From 678f60de741776ef858ac4ae97ba97ce67f87171 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 12 May 2016 14:52:16 +0200 Subject: [PATCH 211/257] GELI: Use a dedicated uma zone for writes to onetime devices ... as they are likely to originate from the vm page daemon. Previously the system could deadlock because the vm daemon was waiting for pages to be written to disk, while GELI was waiting for the vm daemon to make room for the buffer GELI needed to actually write the pages: (kgdb) where #0 sched_switch (td=0xfffff800055bf9a0, newtd=0xfffff80002341000, flags=) at /usr/src/sys/kern/sched_ule.c:1969 #1 0xffffffff80962635 in mi_switch (flags=, newtd=0x0) at /usr/src/sys/kern/kern_synch.c:455 #2 0xffffffff809aaa3a in sleepq_wait (wchan=0x0, pri=0) at /usr/src/sys/kern/subr_sleepqueue.c:637 #3 0xffffffff80962038 in _sleep (ident=, lock=, priority=, wmesg=0xffffffff80e826ee "vmwait", sbt=0, pr=, flags=) at /usr/src/sys/kern/kern_synch.c:229 #4 0xffffffff80c1ac6b in vm_wait () at /usr/src/sys/vm/vm_page.c:2705 #5 0xffffffff80c06a9f in kmem_back (object=0xffffffff8144d6f0, addr=18446741874805047296, size=69632, flags=) at /usr/src/sys/vm/vm_kern.c:356 #6 0xffffffff80c068d2 in kmem_malloc (vmem=0xffffffff813aa500, size=69632, flags=2) at /usr/src/sys/vm/vm_kern.c:316 #7 0xffffffff80bfd7d6 in uma_large_malloc (size=69632, wait=2) at /usr/src/sys/vm/uma_core.c:1106 #8 0xffffffff8092f614 in malloc (size=, mtp=0xffffffff81b4d520, flags=0) at /usr/src/sys/kern/kern_malloc.c:513 #9 0xffffffff81b4ab99 in g_eli_crypto_run (wr=0xfffff80002560040, bp=0xfffff80008a86d90) at /usr/src/sys/modules/geom/geom_eli/../../../geom/eli/g_eli_privacy.c:262 #10 0xffffffff81b3e860 in g_eli_worker (arg=0xfffff80002560040) at /usr/src/sys/modules/geom/geom_eli/../../../geom/eli/g_eli.c:565 #11 0xffffffff80910f5c in fork_exit (callout=0xffffffff81b3e0b0 , arg=0xfffff80002560040, frame=0xfffffe005005ec00) at /usr/src/sys/kern/kern_fork.c:1034 #12 0xffffffff80c33f0e in fork_trampoline () at /usr/src/sys/amd64/amd64/exception.S:611 #13 0x0000000000000000 in ?? () (kgdb) p vm_cnt $16 = {v_swtch = 0, v_trap = 0, v_syscall = 0, v_intr = 0, v_soft = 0, v_vm_faults = 0, v_io_faults = 0, v_cow_faults = 0, v_cow_optim = 0, v_zfod = 0, v_ozfod = 0, v_swapin = 0, v_swapout = 0, v_swappgsin = 0, v_swappgsout = 0, v_vnodein = 0, v_vnodeout = 0, v_vnodepgsin = 0, v_vnodepgsout = 0, v_intrans = 0, v_reactivated = 0, v_pdwakeups = 22197, v_pdpages = 0, v_tcached = 0, v_dfree = 0, v_pfree = 0, v_tfree = 0, v_page_size = 4096, v_page_count = 247688, v_free_reserved = 372, v_free_target = 5320, v_free_min = 1609, v_free_count = 2, v_wire_count = 140735, v_active_count = 96194, v_inactive_target = 7980, v_inactive_count = 10756, v_cache_count = 0, v_pageout_free_min = 34, v_interrupt_free_min = 2, v_free_severe = 990, v_forks = 0, v_vforks = 0, v_rforks = 0, v_kthreads = 0, v_forkpages = 0, v_vforkpages = 0, v_rforkpages = 0, v_kthreadpages = 0, v_spare = 0xffffffff8144d5ac} A sysctl is added to optionally use the zone for GELI writes in general, without letting common writes cut into the reserve for onetime writes. This may reduce latency for larger writes and as we need to keep a couple of items in the zone anyway, the impact on the zone size is minor. Initial testing seems to indicate that the sysctl could be safely enabled by default in the future. Currently a single zone with a somewhat humongous item size sufficient for all GELI writes is being used. While this may look a bit wasteful, in practice we don't need a lot of items, so this seem tolerable for now. The best solution would probably be to only use the dedicated uma zone for common writes if the size is above 65356 bytes, the largest zone item size internally used by malloc. Currently the zone isn't used for reads as those are less time critical and usually are small enough for malloc() to succeed right away anyway. Example length distribution when reproducing ElectroBSD with -j4 and 1 GB of RAM: gpt/swap-ada1.eli BIO_WRITE value ------------- Distribution ------------- count < 4096 | 0 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 4965848 8192 |@@@@@ 943980 12288 |@@ 362668 16384 |@ 161485 20480 |@ 120939 24576 | 87827 28672 | 57402 32768 | 40470 36864 | 42243 40960 | 28543 45056 | 20347 49152 | 15235 53248 | 13450 57344 | 9535 61440 | 9952 65536 |@ 179360 69632 | 0 gpt/swap-ada1.eli BIO_READ value ------------- Distribution ------------- count < 4096 | 0 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 4645114 8192 | 0 12288 | 0 16384 | 3446 20480 | 0 Note that the GELI overhead is not accounted for here and only the results for the swap device are shown. Zone use: [fk@elektrobier3 ~]$ vmstat -z | egrep 'ITEM|eli' | column -t ITEM SIZE LIMIT USED FREE REQ FAIL SLEEP g_eli: 172032, 0, 0, 14, 8077487, 0, 0 This includes writes to gpt/dpool-ada1.eli and gpt/dpool-ada1.eli. Discussion: While the zone served 8077487 memory requests total, 14 items were sufficient for this and therefore the zone only withheld 172032 * 14 bytes plus zone meta data from the rest of the system. Obtained from: ElectroBSD --- sys/geom/eli/g_eli.c | 26 ++++++++++++++++++++++++-- sys/geom/eli/g_eli.h | 18 ++++++++++++++++++ sys/geom/eli/g_eli_privacy.c | 35 +++++++++++++++++++++++++++++++++-- 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index 9add13044a6d..4c6309c9af7e 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -82,6 +82,17 @@ u_int g_eli_batch = 0; SYSCTL_UINT(_kern_geom_eli, OID_AUTO, batch, CTLFLAG_RWTUN, &g_eli_batch, 0, "Use crypto operations batching"); +uma_zone_t g_eli_zone; +static u_int g_eli_uma_reserve = 1; +SYSCTL_UINT(_kern_geom_eli, OID_AUTO, uma_reserve, CTLFLAG_RDTUN, + &g_eli_uma_reserve, 0, "Items to pre-allocate in dedicated uma zone " + "and reserve for writes to onetime disks"); + +u_int g_eli_all_writes_use_uma = 0; +SYSCTL_UINT(_kern_geom_eli, OID_AUTO, use_uma_for_all_writes, CTLFLAG_RDTUN, + &g_eli_all_writes_use_uma, 0, "Use the dedicated uma zone for all writes. " + "May reduce write latency but also inflates memory use a bit"); + /* * Passphrase cached during boot, in order to be more user-friendly if * there are multiple providers using the same passphrase. @@ -246,7 +257,12 @@ g_eli_write_done(struct bio *bp) pbp->bio_inbed++; if (pbp->bio_inbed < pbp->bio_children) return; - free(pbp->bio_driver2, M_ELI); + sc = pbp->bio_to->geom->softc; + if (g_eli_all_writes_use_uma || + (sc->sc_flags & G_ELI_FLAG_ONETIME) != 0) + uma_zfree(g_eli_zone, pbp->bio_driver2); + else + free(pbp->bio_driver2, M_ELI); pbp->bio_driver2 = NULL; if (pbp->bio_error != 0) { G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__, @@ -258,7 +274,6 @@ g_eli_write_done(struct bio *bp) /* * Write is finished, send it up. */ - sc = pbp->bio_to->geom->softc; g_io_deliver(pbp, pbp->bio_error); if (sc != NULL) atomic_subtract_int(&sc->sc_inflight, 1); @@ -1254,6 +1269,12 @@ static void g_eli_init(struct g_class *mp) { + g_eli_zone = uma_zcreate("g_eli", ELI_ZONE_ITEM_SIZE, NULL, NULL, + NULL, NULL, 0, UMA_ZONE_NOFREE); + /* Increase the chances that items are available when needed. */ + uma_prealloc(g_eli_zone, g_eli_uma_reserve); + uma_zone_reserve(g_eli_zone, g_eli_uma_reserve); + g_eli_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync, g_eli_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST); if (g_eli_pre_sync == NULL) @@ -1264,6 +1285,7 @@ static void g_eli_fini(struct g_class *mp) { + uma_zdestroy(g_eli_zone); if (g_eli_pre_sync != NULL) EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_eli_pre_sync); } diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h index 680f67332dd3..4119348f68a6 100644 --- a/sys/geom/eli/g_eli.h +++ b/sys/geom/eli/g_eli.h @@ -139,6 +139,24 @@ #define G_ELI_CRYPTO_SW 2 #ifdef _KERNEL +/* + * Items in the dedicated uma zone have a fixed size and need + * to be big enough for all write lengths. + * + * MAXPHYS is the largest amount of data geli can receive in a row, + * additionally we have to account for the encryption overhead, which + * depends on the number of sectors. + * + * 512 bytes is the smallest sector size supported and results in the + * largest overhead. If larger sectors are being used, we'll just waste + * a bit more memory. + * + * Given that the zone does not need a lot of items, the generous + * item size seems tolerable for now. + */ +#define ELI_ZONE_ITEM_SIZE (MAXPHYS + (MAXPHYS / 512) * \ + (sizeof(struct cryptop) + sizeof(struct cryptodesc))) + extern int g_eli_debug; extern u_int g_eli_overwrites; extern u_int g_eli_batch; diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c index 6ed584684092..282b88627597 100644 --- a/sys/geom/eli/g_eli_privacy.c +++ b/sys/geom/eli/g_eli_privacy.c @@ -49,6 +49,9 @@ __FBSDID("$FreeBSD$"); #include #include +extern u_int g_eli_all_writes_use_uma; +extern uma_zone_t g_eli_zone; + /* * Code paths: * BIO_READ: @@ -153,7 +156,11 @@ g_eli_crypto_write_done(struct cryptop *crp) if (bp->bio_error != 0) { G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).", bp->bio_error); - free(bp->bio_driver2, M_ELI); + if (g_eli_all_writes_use_uma || + (sc->sc_flags & G_ELI_FLAG_ONETIME) != 0) + uma_zfree(g_eli_zone, bp->bio_driver2); + else + free(bp->bio_driver2, M_ELI); bp->bio_driver2 = NULL; g_destroy_bio(cbp); g_io_deliver(bp, bp->bio_error); @@ -259,8 +266,32 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp) */ if (bp->bio_cmd == BIO_WRITE) size += bp->bio_length; - p = malloc(size, M_ELI, M_WAITOK); + if (bp->bio_cmd == BIO_WRITE && + (((sc->sc_flags & G_ELI_FLAG_ONETIME) != 0) || + g_eli_all_writes_use_uma)) { + int uma_flags; + + KASSERT(size <= ELI_ZONE_ITEM_SIZE, + ("Insufficient ELI_ZONE_ITEM_SIZE %u < %u", + (unsigned)ELI_ZONE_ITEM_SIZE, (unsigned)size)); + /* + * Writes to onetime providers are likely to originate + * from the page daemon, therefore we try to get the + * memory a bit harder for them to prevent vm deadlocks. + */ + if ((sc->sc_flags & G_ELI_FLAG_ONETIME) != 0) + uma_flags = M_NOWAIT|M_USE_RESERVE; + else + uma_flags = M_WAITOK; + + while (NULL == (p = uma_zalloc(g_eli_zone, uma_flags))) { + /* Only reachable for onetime providers */ + pause("g_eli:uma", min(hz/1000, 1)); + } + } else { + p = malloc(size, M_ELI, M_WAITOK); + } bp->bio_inbed = 0; bp->bio_children = nsec; bp->bio_driver2 = p; -- 2.11.0 From ec45108878858ab29cb13c1f5548341572c7bcc4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 27 May 2016 18:55:19 +0200 Subject: [PATCH 212/257] atkbdc: Work around (apparently) broken mouse reinitialization ... by increasing pkterrthresh to 10. Previously moving the mouse while loading DTrace modules, for example by running a DTrace script without having loaded the modules manually first, would reliably disables the mouse (track point) until reboot (on a ThinkPad R500). With debug.psm.loglevel=5 the logs showed: Nov 25 13:38:06 r500 kernel: [132] psmintr: 28 03 fe 01 ff 00 Nov 25 13:38:06 r500 kernel: [132] psmintr: 28 02 ff 01 ff 00 Nov 25 13:38:06 r500 kernel: [132] psmintr: 28 03 fe 02 ff 00 Nov 25 13:38:06 r500 kernel: [132] psmintr: 08 02 00 03 fe 00 Nov 25 13:38:06 r500 kernel: [132] psmintr: 28 01 ff 03 fe 00 Nov 25 13:38:07 r500 kernel: [132] psmintr: 18 fe 00 03 ff 00 Nov 25 13:38:08 r500 sudo: fk : TTY=ttyv0 ; PWD=/home/fk ; USER=root ; COMMAND=/usr/share/dtrace/toolkit/execsnoop Nov 25 13:38:08 r500 kernel: [134] psm0: lost interrupt? Nov 25 13:38:09 r500 kernel: [134] psmintr: 08 00 01 03 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 08 00 01 04 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 ff 01 04 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 ff 02 05 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 ff 02 06 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fe 03 05 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fe 04 06 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fd 05 06 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fd 05 03 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fe 03 05 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fd 03 06 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fa 06 05 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fc 04 06 ff 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fa 03 04 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 f9 05 07 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 f7 03 05 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 f9 04 06 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 f8 02 04 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fb 01 04 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 fb 01 02 00 00 Nov 25 13:38:09 r500 kernel: [134] psmintr: 18 ff 00 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 18 ff 00 02 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: c8 d3 0b 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: out of sync (00c0 != 0000) 881 cmds since last error. Nov 25 13:38:11 r500 kernel: [136] psmintr: discard a byte (1) Nov 25 13:38:11 r500 kernel: [136] psmintr: d3 0b 08 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: out of sync (00c0 != 0000) 0 cmds since last error. Nov 25 13:38:11 r500 kernel: [136] psmintr: discard a byte (2) Nov 25 13:38:11 r500 kernel: [136] psmintr: 0b 08 4b 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 3c 08 0c 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0f 08 0c 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 10 08 0b 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 10 08 0a 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 12 08 08 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 10 08 08 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 12 08 08 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 13 08 05 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0f 08 05 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 13 08 04 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 13 08 01 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0b 08 00 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0a 18 ff 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0f 18 fd 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0e 18 fc 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 08 18 fc 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 0a 18 fc 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 05 18 fb 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 05 18 fa 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 05 18 f5 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 09 18 f2 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 08 18 f2 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 05 18 f0 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 06 18 f2 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 03 18 ee 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 02 18 f3 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 01 18 ef 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: 00 38 ed 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: ff 38 ee 03 00 00 Nov 25 13:38:11 r500 kernel: [136] psmintr: out of sync (00c0 != 0000) 0 cmds since last error. Nov 25 13:38:11 r500 kernel: [136] psmintr: reset the mouse. Nov 25 13:38:12 r500 kernel: [137] psm0: current command byte: 0047 (reinitialize). Nov 25 13:38:12 r500 kernel: [137] psm: DISABLE_DEV return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: ENABLE_DEV return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: DISABLE_DEV return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: SET_SAMPLING_RATE (100) 00fa Nov 25 13:38:12 r500 kernel: [137] psm: SET_RESOLUTION (2) 00fa Nov 25 13:38:12 r500 kernel: [137] psm: SET_SCALING11 return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: SET_STREAM_MODE return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: SEND_AUX_DEV_STATUS return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: status 00 02 64 Nov 25 13:38:12 r500 kernel: [137] psm: ENABLE_DEV return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: SEND_AUX_DEV_STATUS return code:00fa Nov 25 13:38:12 r500 kernel: [137] psm: status 20 02 64 Nov 25 13:38:15 r500 kernel: [140] psm0: lost interrupt? Nov 25 13:38:16 r500 kernel: [141] psm0: lost interrupt? Nov 25 13:38:17 r500 kernel: [142] psm0: lost interrupt? Nov 25 13:38:18 r500 kernel: [143] psm0: lost interrupt? Nov 25 13:38:19 r500 kernel: [144] psm0: lost interrupt? Nov 25 13:38:20 r500 kernel: [145] psm0: lost interrupt? Nov 25 13:38:21 r500 kernel: [147] psm0: lost interrupt? Nov 25 13:38:22 r500 kernel: [148] psm0: lost interrupt? Nov 25 13:38:23 r500 kernel: [149] psm0: lost interrupt? Nov 25 13:38:24 r500 kernel: [150] psm0: lost interrupt? Nov 25 13:38:25 r500 kernel: [151] psm0: lost interrupt? Nov 25 13:38:26 r500 kernel: [152] psm0: lost interrupt? Nov 25 13:38:27 r500 kernel: [153] psm0: lost interrupt? Nov 25 13:38:29 r500 kernel: [154] psm0: lost interrupt? Nov 25 13:38:30 r500 kernel: [155] psm0: lost interrupt? Nov 25 13:38:31 r500 kernel: [156] psm0: lost interrupt? Nov 25 13:38:32 r500 kernel: [157] psm0: lost interrupt? Nov 25 13:38:33 r500 kernel: [158] psm0: lost interrupt? Nov 25 13:38:34 r500 kernel: [159] psm0: lost interrupt? Nov 25 13:38:35 r500 kernel: [160] psm0: lost interrupt? Nov 25 13:38:36 r500 kernel: [161] psm0: lost interrupt? Nov 25 13:38:37 r500 kernel: [162] psm0: lost interrupt? Nov 25 13:38:38 r500 kernel: [163] psm0: lost interrupt? Nov 25 13:38:39 r500 kernel: [164] psm0: lost interrupt? Nov 25 13:38:40 r500 kernel: [165] psm0: lost interrupt? After the "reset" the mouse cursor no longer moved and rebooting seemed to be the only "cure". Setting debug.psm.pkterrthresh=10 seems to work around the issue, so does not moving the cursor until the modules are loaded. Mouse resets for other reasons seemed to occasionally render the mouse useless as well, but triggering the problem by loading DTrace modules is the only reproducible method currently known. Obtained from: ElectroBSD --- sys/dev/atkbdc/psm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/atkbdc/psm.c b/sys/dev/atkbdc/psm.c index e5f9838c91ba..234d55806a71 100644 --- a/sys/dev/atkbdc/psm.c +++ b/sys/dev/atkbdc/psm.c @@ -2457,7 +2457,7 @@ SYSCTL_INT(_debug_psm, OID_AUTO, secs, CTLFLAG_RW, &psmsecs, 0, static int psmusecs = 500000; SYSCTL_INT(_debug_psm, OID_AUTO, usecs, CTLFLAG_RW, &psmusecs, 0, "Microseconds to add to psmsecs"); -static int pkterrthresh = 2; +static int pkterrthresh = 10; SYSCTL_INT(_debug_psm, OID_AUTO, pkterrthresh, CTLFLAG_RW, &pkterrthresh, 0, "Number of error packets allowed before reinitializing the mouse"); -- 2.11.0 From a6767cf581a44fb7e7bcdb11ae34135f897c2fb2 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 16:36:46 +0200 Subject: [PATCH 213/257] vm_pageout_scan(): Add SDT probes to make a couple of internal variables visible Obtained from: ElectroBSD --- sys/vm/vm_pageout.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 67cf415d2e1e..3e36f7c1c3fa 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -140,6 +140,15 @@ SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start, SDT_PROVIDER_DEFINE(vm); SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan); +SDT_PROBE_DEFINE4(vm, , , before__inactive__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage", "int deficit"); +SDT_PROBE_DEFINE5(vm, , , after__inactive__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage", "int addl_page_shortage", + "int vnodes_skipped"); +SDT_PROBE_DEFINE3(vm, , , before__active__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage"); +SDT_PROBE_DEFINE3(vm, , , after__active__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage"); #if !defined(NO_SWAPPING) /* the kernel process "vm_daemon"*/ @@ -921,6 +930,9 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) vnodes_skipped = 0; + SDT_PROBE4(vm, , , before__inactive__scan, vmd, pass, page_shortage, + deficit); + /* * Start scanning the inactive queue for pages that we can free. The * scan will stop when we reach the target or we have scanned the @@ -1146,6 +1158,9 @@ relock_queue: } vm_pagequeue_unlock(pq); + SDT_PROBE5(vm, , , after__inactive__scan, vmd, pass, page_shortage, + addl_page_shortage, vnodes_skipped); + #if !defined(NO_SWAPPING) /* * Wakeup the swapout daemon if we didn't free the targeted number of @@ -1194,6 +1209,8 @@ relock_queue: if (min_scan > 0 || (inactq_shortage > 0 && maxscan > 0)) vmd->vmd_last_active_scan = scan_tick; + SDT_PROBE3(vm, , , before__active__scan, vmd, pass, page_shortage); + /* * Scan the active queue for pages that can be deactivated. Update * the per-page activity counter and use it to identify deactivation @@ -1271,6 +1288,9 @@ relock_queue: vm_page_unlock(m); } vm_pagequeue_unlock(pq); + + SDT_PROBE3(vm, , , after__active__scan, vmd, pass, page_shortage); + #if !defined(NO_SWAPPING) /* * Idle process swapout -- run once per second. -- 2.11.0 From d7b3c813d212598f5b6a416617ba0a3fc0e8297f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 24 Jun 2016 14:39:28 +0200 Subject: [PATCH 214/257] sys/amd64/conf/ELECTRO_BLOAT: Remove COMPAT support for FreeBSD versions before 10 Obtained from: ElectroBSD --- sys/amd64/conf/ELECTRO_BLOAT | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sys/amd64/conf/ELECTRO_BLOAT b/sys/amd64/conf/ELECTRO_BLOAT index a61f4e5eee3a..3bd60d2df610 100644 --- a/sys/amd64/conf/ELECTRO_BLOAT +++ b/sys/amd64/conf/ELECTRO_BLOAT @@ -48,11 +48,6 @@ options GEOM_PART_GPT # GUID Partition Tables. options GEOM_RAID # Soft RAID functionality. options GEOM_LABEL # Provides labelization options COMPAT_FREEBSD32 # Compatible with i386 binaries -options COMPAT_FREEBSD4 # Compatible with FreeBSD4 -options COMPAT_FREEBSD5 # Compatible with FreeBSD5 -options COMPAT_FREEBSD6 # Compatible with FreeBSD6 -options COMPAT_FREEBSD7 # Compatible with FreeBSD7 -options COMPAT_FREEBSD9 # Compatible with FreeBSD9 options COMPAT_FREEBSD10 # Compatible with FreeBSD10 options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI options KTRACE # ktrace(1) support -- 2.11.0 From 8bb525e8c856c78d0cdb9c2ac0ac7ec714a7b50e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 13 Feb 2015 17:56:07 +0100 Subject: [PATCH 215/257] range_tree_destroy(): Optionally tolerate non-zero rt->rt_space Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c index 6422fd1c1fa6..842480a9e4ad 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c @@ -150,7 +150,11 @@ range_tree_create(range_tree_ops_t *ops, void *arg, kmutex_t *lp) void range_tree_destroy(range_tree_t *rt) { - VERIFY0(rt->rt_space); + if (rt->rt_space != 0) { + zfs_panic_recover("zfs: range_tree_destroy(): " + "rt->rt_space != 0: %llx", rt->rt_space); + return; + } if (rt->rt_ops != NULL) rt->rt_ops->rtop_destroy(rt, rt->rt_arg); -- 2.11.0 From ed06783324cc88cf4d13d7e2bfd588c1a1701601 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 17 Jul 2015 13:11:39 +0200 Subject: [PATCH 216/257] Let the ZFS ARC behave better under memory pressure Original author: Karl Denninger Source: https://bz-attachments.freebsd.org/attachment.cgi?id=152852&action=diff&collapsed=&context=patch&format=raw&headers=1 PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=187594 Commit adjusted to compile on i386 and to deal with merge conflicts with (at least): r263620/6fcf6199a: "Rename global cnt to vm_cnt to avoid shadowing" r286625/78648874e: "5376 arc_kmem_reap_now() should not result in clearing arc_no_grow" r286763/71fb6300f: "5497 lock contention on arcs_mtx" No functional change intended. --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 82 +++++++++++++++++----- .../contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 47 +++++++++++-- .../opensolaris/uts/common/fs/zfs/dsl_pool.c | 7 +- .../opensolaris/uts/common/fs/zfs/sys/dsl_pool.h | 1 + .../contrib/opensolaris/uts/common/fs/zfs/zio.c | 4 +- 5 files changed, 116 insertions(+), 25 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 6e9a624513cf..613a4ebdf1d1 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -341,6 +341,15 @@ static int arc_dead; extern boolean_t zfs_prefetch_disable; /* + * KD 2015-02-10 + * We have to be able to test for UIO use inside the arc allocator. + * NOTE: DO NOT MODIFY HERE! + */ +extern int zio_use_uma; +extern int zfs_dynamic_write_buffer; + + +/* * The arc has filled available memory and has now warmed up. */ static boolean_t arc_warm; @@ -373,7 +382,7 @@ static void arc_free_target_init(void *unused __unused) { - zfs_arc_free_target = vm_pageout_wakeup_thresh; + zfs_arc_free_target = vm_pageout_wakeup_thresh + ((vm_cnt.v_free_target - vm_pageout_wakeup_thresh) / 2); } SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, arc_free_target_init, NULL); @@ -394,6 +403,9 @@ SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_shrink_shift, CTLFLAG_RW, "log2(fraction of arc to reclaim)"); SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RDTUN, &zfs_compressed_arc_enabled, 0, "Enable compressed ARC"); +SYSCTL_INT(_vfs_zfs, OID_AUTO, dynamic_write_buffer, CTLFLAG_RWTUN, + &zfs_dynamic_write_buffer, 0, + "Dynamically restrict dirty data when memory is low"); /* * We don't have a tunable for arc_free_target due to the dependency on @@ -4011,13 +4023,32 @@ extern kmem_cache_t *zio_buf_cache[]; extern kmem_cache_t *zio_data_buf_cache[]; extern kmem_cache_t *range_seg_cache; -static __noinline void -arc_kmem_reap_now(void) +static void __used +reap_arc_caches() { size_t i; kmem_cache_t *prev_cache = NULL; kmem_cache_t *prev_data_cache = NULL; + for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { + if (zio_buf_cache[i] != prev_cache) { + prev_cache = zio_buf_cache[i]; + kmem_cache_reap_now(zio_buf_cache[i]); + } + if (zio_data_buf_cache[i] != prev_data_cache) { + prev_data_cache = zio_data_buf_cache[i]; + kmem_cache_reap_now(zio_data_buf_cache[i]); + } + } + kmem_cache_reap_now(buf_cache); + kmem_cache_reap_now(hdr_full_cache); + kmem_cache_reap_now(hdr_l2only_cache); + kmem_cache_reap_now(range_seg_cache); +} + +static __noinline void +arc_kmem_reap_now(void) +{ DTRACE_PROBE(arc__kmem_reap_start); #ifdef _KERNEL if (arc_meta_used >= arc_meta_limit) { @@ -4035,20 +4066,7 @@ arc_kmem_reap_now(void) #endif #endif - for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { - if (zio_buf_cache[i] != prev_cache) { - prev_cache = zio_buf_cache[i]; - kmem_cache_reap_now(zio_buf_cache[i]); - } - if (zio_data_buf_cache[i] != prev_data_cache) { - prev_data_cache = zio_data_buf_cache[i]; - kmem_cache_reap_now(zio_data_buf_cache[i]); - } - } - kmem_cache_reap_now(buf_cache); - kmem_cache_reap_now(hdr_full_cache); - kmem_cache_reap_now(hdr_l2only_cache); - kmem_cache_reap_now(range_seg_cache); + reap_arc_caches(); #ifdef illumos if (zio_arena != NULL) { @@ -4083,11 +4101,28 @@ arc_reclaim_thread(void *dummy __unused) { hrtime_t growtime = 0; callb_cpr_t cpr; + int autoreap = 0; CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG); mutex_enter(&arc_reclaim_lock); while (!arc_reclaim_thread_exit) { +#ifdef _KERNEL +/* KD 2015-02-10 + * Protect against UMA free memory bloat. We already do this on a low-memory + * basis in the allocator; it has to happen there rather than here due to + * response time considerations. Make the call here once every 10 passes as + * well; this reclaims unused UMA buffers every 10 seconds on an idle system + * and more frequently if the reclaim thread gets woken up by low RAM + * conditions. + */ + if ((zio_use_uma) && (autoreap++ == 10)) { + autoreap = 0; + DTRACE_PROBE(arc__reclaim_timed_reap); + reap_arc_caches(); + } +#endif /* _KERNEL */ + int64_t free_memory = arc_available_memory(); uint64_t evicted = 0; @@ -4373,6 +4408,19 @@ arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag) arc_space_consume(size, ARC_SPACE_META); } else { ASSERT(type == ARC_BUFC_DATA); +#ifdef _KERNEL +/* KD 2015-02-10 + * It would be nice if we could leave this to the arc_reclaim thread. + * Unfortunately we cannot; the test has to be done here as well, because + * under heavy I/O demand we can grab enough RAM fast enough to induce + * nasty oscillation problems. Fortunately we only need to call this when + * the system is under reasonably-severe memory stress. + */ + if (zio_use_uma && (ptob(vm_cnt.v_free_count) + size < ptob(vm_cnt.v_free_target))) { + DTRACE_PROBE3(arc__alloc_lowmem_reap, int, vm_cnt.v_free_count, int, size, int, vm_cnt.v_free_target); + reap_arc_caches(); + } +#endif /* _KERNEL */ datap = zio_data_buf_alloc(size); arc_space_consume(size, ARC_SPACE_DATA); } diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 68381488b38c..5bcc2cec49e3 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -43,6 +43,8 @@ typedef void (*dmu_tx_hold_func_t)(dmu_tx_t *tx, struct dnode *dn, uint64_t arg1, uint64_t arg2); +extern int zio_use_uma; /* Needs to be visible; DO NOT MODIFY! */ +int zfs_dynamic_write_buffer = 1; /* Dynamically tune writes */ dmu_tx_t * dmu_tx_create_dd(dsl_dir_t *dd) @@ -1106,7 +1108,7 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) { dsl_pool_t *dp = tx->tx_pool; uint64_t delay_min_bytes = - zfs_dirty_data_max * zfs_delay_min_dirty_percent / 100; + zfs_dirty_data_max_internal * zfs_delay_min_dirty_percent / 100; hrtime_t wakeup, min_tx_time, now; if (dirty <= delay_min_bytes) @@ -1118,11 +1120,11 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) * have to handle the case of it being >= the max, which could * cause a divide-by-zero if it's == the max. */ - ASSERT3U(dirty, <, zfs_dirty_data_max); + ASSERT3U(dirty, <, zfs_dirty_data_max_internal); now = gethrtime(); min_tx_time = zfs_delay_scale * - (dirty - delay_min_bytes) / (zfs_dirty_data_max - dirty); + (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); if (now > tx->tx_start + min_tx_time) return; @@ -1327,6 +1329,7 @@ int dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) { int err; + static uint64_t last_max; ASSERT(tx->tx_txg == 0); ASSERT(txg_how == TXG_WAIT || txg_how == TXG_NOWAIT || @@ -1339,6 +1342,42 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) if (txg_how == TXG_WAITED) tx->tx_waited = B_TRUE; +#ifdef _KERNEL + /* + * KD 2014-09-22 + * If UMA is enabled it can only return a previously-used block + * of identical size to what it had out before. If it's not the + * same size it will allocate a new one. This is a problem because + * dirty_data_max is the total dirty write data allowed out at any + * given time, but with UMA on that can multiply by the number of + * different block sizes (!!) requested in terms of free RAM that + * is left allocated but unused. For this reason never allow + * dirty_data_max to exceed the difference between the paging + * threshold and the current free memory, with a minimum of 256MB. + * This throttles "burst" allocations and prevents the system from + * choking during times of high write I/O demand. + * + * We allow this to be turned off if you want with + * "vfs.zfs_dynamic_write_buffer=0", which can be done in real time. + * + * Note that we work on the zfs_dirty_data_max_internal variable, + * because the user may set zfs_dirty_data_max himself and we must + * must honor that as a hard cap so it remains a usable tunable value. + */ + if (zio_use_uma & zfs_dynamic_write_buffer) { + zfs_dirty_data_max_internal = 1 << 28; + zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max); + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max_max); + if (last_max != (zfs_dirty_data_max_internal / (1024 * 1024))) { + last_max = zfs_dirty_data_max_internal / (1024 * 1024); + DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); + } + } else { + zfs_dirty_data_max_internal = zfs_dirty_data_max; + } +#endif /* _KERNEL */ + while ((err = dmu_tx_try_assign(tx, txg_how)) != 0) { dmu_tx_unassign(tx); @@ -1369,7 +1408,7 @@ dmu_tx_wait(dmu_tx_t *tx) * space. */ mutex_enter(&dp->dp_lock); - while (dp->dp_dirty_total >= zfs_dirty_data_max) + while (dp->dp_dirty_total >= zfs_dirty_data_max_internal) cv_wait(&dp->dp_spaceavail_cv, &dp->dp_lock); uint64_t dirty = dp->dp_dirty_total; mutex_exit(&dp->dp_lock); diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c index 9b3b79bfb517..33820f76e7b3 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c @@ -99,8 +99,11 @@ /* * zfs_dirty_data_max will be set to zfs_dirty_data_max_percent% of all memory, * capped at zfs_dirty_data_max_max. It can also be overridden in /etc/system. + * We also dynamically tune during low memory, honoring the sysctl set, so + * internal comparisons are against zfs_dirty_data_max_internal. */ uint64_t zfs_dirty_data_max; +uint64_t zfs_dirty_data_max_internal; uint64_t zfs_dirty_data_max_max = 4ULL * 1024 * 1024 * 1024; int zfs_dirty_data_max_percent = 10; @@ -548,7 +551,7 @@ dsl_pool_dirty_delta(dsl_pool_t *dp, int64_t delta) * Note: we signal even when increasing dp_dirty_total. * This ensures forward progress -- each thread wakes the next waiter. */ - if (dp->dp_dirty_total <= zfs_dirty_data_max) + if (dp->dp_dirty_total <= zfs_dirty_data_max_internal) cv_signal(&dp->dp_spaceavail_cv); } @@ -730,7 +733,7 @@ boolean_t dsl_pool_need_dirty_delay(dsl_pool_t *dp) { uint64_t delay_min_bytes = - zfs_dirty_data_max * zfs_delay_min_dirty_percent / 100; + zfs_dirty_data_max_internal * zfs_delay_min_dirty_percent / 100; boolean_t rv; mutex_enter(&dp->dp_lock); diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h index 0e27a538208c..ee97b5719cd2 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h @@ -50,6 +50,7 @@ struct dmu_tx; struct dsl_scan; extern uint64_t zfs_dirty_data_max; +extern uint64_t zfs_dirty_data_max_internal; extern uint64_t zfs_dirty_data_max_max; extern uint64_t zfs_dirty_data_sync; extern int zfs_dirty_data_max_percent; diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c index f8907c1e6872..d5ab54271619 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c @@ -46,9 +46,9 @@ SYSCTL_DECL(_vfs_zfs); SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); #if defined(__amd64__) -static int zio_use_uma = 1; +int zio_use_uma = 1; #else -static int zio_use_uma = 0; +int zio_use_uma = 0; #endif SYSCTL_INT(_vfs_zfs_zio, OID_AUTO, use_uma, CTLFLAG_RDTUN, &zio_use_uma, 0, "Use uma(9) for ZIO allocations"); -- 2.11.0 From e821063f33fcae90fc6ed559f0be5c182f5c9229 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 11:33:46 +0200 Subject: [PATCH 217/257] ZFS ARC: Add a missing & to get a logical and instead of a binary one Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 5bcc2cec49e3..6e89c7d3b6a5 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1364,7 +1364,7 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) * because the user may set zfs_dirty_data_max himself and we must * must honor that as a hard cap so it remains a usable tunable value. */ - if (zio_use_uma & zfs_dynamic_write_buffer) { + if (zio_use_uma && zfs_dynamic_write_buffer) { zfs_dirty_data_max_internal = 1 << 28; zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max); -- 2.11.0 From 0cc5f291e6632fc5838a472060ff9569e8ebe113 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 3 Sep 2015 13:58:29 +0200 Subject: [PATCH 218/257] ZFS ARC: Move the ARC cache reaping out of the hot path ... and use a time-based rate limiting approach that can be controlled with the vfs.zfs.arc_reap_delay_min sysctl. Previously memory pressure could result in several thousand ARC reaping attempts per second. This wasn't merely inefficient but also caused lock contention and poor latency for ZFS operations that relied on arc_get_data_buf(). For details see: https://www.fabiankeil.de/gehacktes/electrobsd/zfs-arc-tuning/ Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 88 ++++++++++++++++------ 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 613a4ebdf1d1..8886ddcfeb89 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -4023,8 +4023,21 @@ extern kmem_cache_t *zio_buf_cache[]; extern kmem_cache_t *zio_data_buf_cache[]; extern kmem_cache_t *range_seg_cache; -static void __used -reap_arc_caches() +/* + * Used by arc_kmem_reap_now() and consider_reaping_arc_caches() + * to limit the time spent reaping. + * + * The arc_reaping_in_progress is a (somewhat racy) left over from a + * previous version of this code which could trigger multiple ARC cache + * reapings in parallel which should be avoided to reduce lock + * contention. It's hasn't been removed yet to encourage further + * experimenting. + */ +static unsigned int arc_reaping_in_progress = 0; +static sbintime_t last_reaping = 0; + +static void __noinline +reap_arc_caches(void) { size_t i; kmem_cache_t *prev_cache = NULL; @@ -4050,6 +4063,8 @@ static __noinline void arc_kmem_reap_now(void) { DTRACE_PROBE(arc__kmem_reap_start); + arc_reaping_in_progress++; + #ifdef _KERNEL if (arc_meta_used >= arc_meta_limit) { /* @@ -4077,9 +4092,49 @@ arc_kmem_reap_now(void) vmem_qcache_reap(zio_arena); } #endif +#ifdef _KERNEL + last_reaping = getsbinuptime(); +#endif + arc_reaping_in_progress = 0; DTRACE_PROBE(arc__kmem_reap_end); } + +/* + * Declared writable to allow resetting it. + * XXX: Should probably be a uint64 and integrated with kstat. + */ +static unsigned int arc_cache_reapings_skipped = 0; +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reapings_skipped, CTLFLAG_RW, + &arc_cache_reapings_skipped, 0, "Number of times the ARC caches have not been reaped due to the reap delay"); + +static unsigned int min_arc_reap_delay = 200; +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_reap_delay_min, CTLFLAG_RW, + &min_arc_reap_delay, 200, "Minimum delay between ARC cache reapings (milliseconds)"); + +static void __noinline +consider_reaping_arc_caches(void) +{ +#ifdef _KERNEL + sbintime_t now; + + if (arc_reaping_in_progress) { + /* Already reaping in another thread. */ + arc_cache_reapings_skipped++; + return; + } + + now = getsbinuptime(); + if ((now - last_reaping) / SBT_1MS < min_arc_reap_delay) + { + /* Too soon to reap again. */ + arc_cache_reapings_skipped++; + return; + } +#endif + arc_kmem_reap_now(); +} + /* * Threads can block in arc_get_data_buf() waiting for this thread to evict * enough data and signal them to proceed. When this happens, the threads in @@ -4101,28 +4156,11 @@ arc_reclaim_thread(void *dummy __unused) { hrtime_t growtime = 0; callb_cpr_t cpr; - int autoreap = 0; CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG); mutex_enter(&arc_reclaim_lock); while (!arc_reclaim_thread_exit) { -#ifdef _KERNEL -/* KD 2015-02-10 - * Protect against UMA free memory bloat. We already do this on a low-memory - * basis in the allocator; it has to happen there rather than here due to - * response time considerations. Make the call here once every 10 passes as - * well; this reclaims unused UMA buffers every 10 seconds on an idle system - * and more frequently if the reclaim thread gets woken up by low RAM - * conditions. - */ - if ((zio_use_uma) && (autoreap++ == 10)) { - autoreap = 0; - DTRACE_PROBE(arc__reclaim_timed_reap); - reap_arc_caches(); - } -#endif /* _KERNEL */ - int64_t free_memory = arc_available_memory(); uint64_t evicted = 0; @@ -4153,8 +4191,6 @@ arc_reclaim_thread(void *dummy __unused) */ growtime = gethrtime() + SEC2NSEC(arc_grow_retry); - arc_kmem_reap_now(); - /* * If we are still low on memory, shrink the ARC * so that we have arc_shrink_min free space. @@ -4177,6 +4213,16 @@ arc_reclaim_thread(void *dummy __unused) evicted = arc_adjust(); + /* + * XXX: 2016-10-10: This was moved from arc_user_evicts_thread() + * which got deleted by the commit to keep the + * ARC data compressed. + * + * Consider reaping the ARC caches at least once per + * second, but more often when signalled under pressure. + */ + consider_reaping_arc_caches(); + mutex_enter(&arc_reclaim_lock); /* -- 2.11.0 From 7d82647c6ce15d5d3aeda68cdefcb0470da33476 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 7 Oct 2015 13:12:26 +0200 Subject: [PATCH 219/257] ZFS ARC: Ignore the reap delay when under memory pressure Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 8886ddcfeb89..55a1ca4c1ff3 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -4107,6 +4107,9 @@ arc_kmem_reap_now(void) static unsigned int arc_cache_reapings_skipped = 0; SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reapings_skipped, CTLFLAG_RW, &arc_cache_reapings_skipped, 0, "Number of times the ARC caches have not been reaped due to the reap delay"); +static unsigned int arc_cache_reapings_forced = 0; +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reapings_forced, CTLFLAG_RW, + &arc_cache_reapings_forced, 0, "Number of times the ARC caches reap delay was ignored due to memory pressure"); static unsigned int min_arc_reap_delay = 200; SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_reap_delay_min, CTLFLAG_RW, @@ -4125,11 +4128,17 @@ consider_reaping_arc_caches(void) } now = getsbinuptime(); - if ((now - last_reaping) / SBT_1MS < min_arc_reap_delay) - { - /* Too soon to reap again. */ - arc_cache_reapings_skipped++; - return; + if ((now - last_reaping) / SBT_1MS < min_arc_reap_delay) { + /* + * Skip the reaping unless there's memory pressure + * in which case we would risk vm deadlocks. We don't + * use vm_page_count_severe() as it triggers too late. + */ + if (!vm_page_count_min()) { + arc_cache_reapings_skipped++; + return; + } + arc_cache_reapings_forced++; } #endif arc_kmem_reap_now(); -- 2.11.0 From 488188f4f79de0fec0570bf2dfad5ffb7a5b2eaf Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 20 Sep 2015 19:53:09 +0200 Subject: [PATCH 220/257] top: Show ZFS ARC target size Obtained from: ElectroBSD --- usr.bin/top/machine.c | 6 ++++-- usr.bin/top/top.local.1 | 12 +++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/usr.bin/top/machine.c b/usr.bin/top/machine.c index 492a62ae227b..597d41fd801d 100644 --- a/usr.bin/top/machine.c +++ b/usr.bin/top/machine.c @@ -182,9 +182,9 @@ char *memorynames[] = { "K Free", NULL }; -int arc_stats[7]; +int arc_stats[8]; char *arcnames[] = { - "K Total, ", "K MFU, ", "K MRU, ", "K Anon, ", "K Header, ", "K Other", + "K Total, ", "K MFU, ", "K MRU, ", "K Anon, ", "K Header, ", "K Other, ", "K Target", NULL }; @@ -557,6 +557,8 @@ get_system_info(struct system_info *si) arc_stats[4] = arc_stat + arc_stat2 >> 10; GETSYSCTL("kstat.zfs.misc.arcstats.other_size", arc_stat); arc_stats[5] = arc_stat >> 10; + GETSYSCTL("kstat.zfs.misc.arcstats.c", arc_stat); + arc_stats[6] = arc_stat >> 10; si->arc = arc_stats; } diff --git a/usr.bin/top/top.local.1 b/usr.bin/top/top.local.1 index 864ab8d65c4b..951754c83bbe 100644 --- a/usr.bin/top/top.local.1 +++ b/usr.bin/top/top.local.1 @@ -2,9 +2,9 @@ .SH "FreeBSD NOTES" .SH DESCRIPTION OF MEMORY -Mem: 9220K Active, 1M Inact, 3284K Wired, 1M Cache, 2M Buf, 1320K Free -ARC: 2048K Total, 342K MRU, 760K MFU, 272K Anon, 232K Header, 442K Other -Swap: 91M Total, 79M Free, 13% Inuse, 80K In, 104K Out +Mem: 387M Active, 499M Inact, 945M Wired, 8K Cache, 5528K Buf, 83M Free +ARC: 349M Total, 174M MFU, 94M MRU, 2192K Anon, 6488K Header, 73M Other, 350M Target +Swap: 2048M Total, 124M Used, 1924M Free, 6% Inuse .TP .B K: Kilobyte @@ -57,6 +57,12 @@ number of ARC bytes holding headers .TP .B Other miscellaneous ARC bytes +.TP +.B Target +ARC target size, that is the total amount of memory +the ARC considers usable for itself. If it's not equal +to the total size, the ARC will shrink or grow to reach +the target. .SS Swap Stats .TP .B Total: -- 2.11.0 From 8151134da29603cdb95eec16d28e29c39a321c68 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 25 Oct 2015 19:49:47 +0100 Subject: [PATCH 221/257] ZFS dsl_scan_visitds(): Don't panic if a device disappears while scrubbing Prevents: Unread portion of the kernel message buffer: [4299] panic: solaris assert: dmu_objset_find_dp(dp, dp->dp_root_dir_obj, enqueue_clones_cb, &eca, (1<<1)) == 0 (0x6 == 0x0), file: /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c, line: 1130 [4299] cpuid = 1 [4299] KDB: stack backtrace: [4299] db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe00949ed310 [4299] vpanic() at vpanic+0x182/frame 0xfffffe00949ed390 [4299] panic() at panic+0x43/frame 0xfffffe00949ed3f0 [4299] zfs_kmem_alloc() at zfs_kmem_alloc/frame 0xfffffe00949ed440 [4299] dsl_scan_visitds() at dsl_scan_visitds+0x551/frame 0xfffffe00949ed570 [4299] dsl_scan_visit() at dsl_scan_visit+0x22e/frame 0xfffffe00949ed790 [4299] dsl_scan_sync() at dsl_scan_sync+0x9da/frame 0xfffffe00949ed920 [4299] spa_sync() at spa_sync+0x564/frame 0xfffffe00949eda90 [4299] txg_sync_thread() at txg_sync_thread+0x3f1/frame 0xfffffe00949edbb0 [4299] fork_exit() at fork_exit+0x9c/frame 0xfffffe00949edbf0 [4299] fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00949edbf0 [4299] --- trap 0, rip = 0, rsp = 0, rbp = 0 --- [4299] KDB: enter: panic Note that this is not the only place where dmu_objset_find_dp() failures lead to panics and ENXIO probably isn't the only return code we have to expect anyway. Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c index 4ebbc1a42050..a00cab366a4f 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c @@ -1179,12 +1179,14 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) scn->scn_phys.scn_queue_obj, dsl_dataset_phys(ds)->ds_creation_txg, tx)); } else { + int error; struct enqueue_clones_arg eca; eca.tx = tx; eca.originobj = ds->ds_object; - VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, - enqueue_clones_cb, &eca, DS_FIND_CHILDREN)); + error = dmu_objset_find_dp(dp, dp->dp_root_dir_obj, + enqueue_clones_cb, &eca, DS_FIND_CHILDREN); + VERIFY(error == 0 || error == ENXIO); } } -- 2.11.0 From 1a605dee8ae09b81f563a26e494e62a11f3a4cb7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 27 Oct 2015 23:09:03 +0100 Subject: [PATCH 222/257] ZFS: Optionally let spa_sync() wait until at least one vdev is writable ... before calling dsl_pool_sync(). Significantly reduces the chances that a temporary loss of all vdevs results in a spa_namespace_lock deadlock that causes the pool to become inaccesible until the next reboot if the failmode is "wait" (default). The deadlock is caused by spa_sync() holding the required locks to fault or clear the pool while waiting for the sync operation to complete, thus preventing other parts of the system to get the pool in a usable state again. As a side effect of the deadlock, various zfs and zpool commands can lock up as well. Obtained from: ElectroBSD Deadlock example: 6 102135 zfskern txg_thread_enter mi_switch+0xde sleepq_wait+0x3a _cv_wait+0x1a6 zio_wait+0x89 dsl_pool_sync+0x12e spa_sync+0x51f txg_sync_thread+0x408 fork_exit+0x9a fork_trampoline+0xe 5985 101966 zpool - mi_switch+0xde sleepq_wait+0x3a _sx_xlock_hard+0x540 _sx_xlock+0x5d spa_get_errlog_size+0x3a spa_get_stats+0x116 zfs_ioc_pool_stats+0x4c zfsdev_ioctl+0xc68 devfs_ioctl_f+0x13b kern_ioctl+0x401 sys_ioctl+0x153 amd64_syscall+0x3e7 Xfast_syscall+0xfb 6863 101673 zpool - mi_switch+0xde sleepq_wait+0x3a _cv_wait+0x1a6 zio_wait+0x89 dbuf_read+0x464 dmu_buf_hold+0x9a zap_get_leaf_byblk+0x76 zap_deref_leaf+0xc6 fzap_cursor_retrieve+0x169 zap_cursor_retrieve+0x216 process_error_log+0xb1 spa_get_errlog+0xae zfs_ioc_error_log+0x70 zfsdev_ioctl+0xc68 devfs_ioctl_f+0x13b kern_ioctl+0x401 sys_ioctl+0x153 amd64_syscall+0x3e7 6885 100784 zpool - mi_switch+0xde sleepq_wait+0x3a _sx_xlock_hard+0x540 _sx_xlock+0x5d spa_get_errlog_size+0x3a spa_get_stats+0x116 zfs_ioc_pool_stats+0x4c zfsdev_ioctl+0xc68 devfs_ioctl_f+0x13b kern_ioctl+0x401 sys_ioctl+0x153 amd64_syscall+0x3e7 Xfast_syscall+0xfb See also: https://lists.freebsd.org/pipermail/freebsd-current/2014-September/052018.html Log excerpt with the sysctl enabled (single-disk pool): Oct 28 12:27:03 r500 kernel: [415] GEOM_ELI: g_eli_read_done() failed (error=5) label/prot1.eli[READ(offset=4654291968, length=9216)] Oct 28 12:27:03 r500 kernel: [415] da0 at umass-sim0 bus 0 scbus2 target 0 lun 0 Oct 28 12:27:03 r500 kernel: [415] da0: s/n 22B39DCC detached Oct 28 12:27:03 r500 kernel: [415] GEOM_ELI: Device label/prot1.eli destroyed. Oct 28 12:27:03 r500 kernel: [415] GEOM_ELI: Detached label/prot1.eli on last close. Oct 28 12:27:03 r500 kernel: [415] (da0:umass-sim0:0:0:0): Periph destroyed Oct 28 12:27:03 r500 ZFS: vdev is removed, pool_guid=2852801396507945726 vdev_guid=18325185749120546863 Oct 28 12:27:04 r500 kernel: [416] spa_sync: No writable vdev for prot1. Oct 28 12:27:05 r500 kernel: [417] spa_sync: No writable vdev for prot1. Oct 28 12:27:06 r500 kernel: [418] spa_sync: No writable vdev for prot1. Oct 28 12:27:07 r500 kernel: [419] spa_sync: No writable vdev for prot1. Oct 28 12:27:08 r500 kernel: [420] spa_sync: No writable vdev for prot1. Oct 28 12:27:09 r500 kernel: [420] umass0: on usbus1 Oct 28 12:27:09 r500 kernel: [420] umass0: SCSI over Bulk-Only; quirks = 0x4101 Oct 28 12:27:09 r500 kernel: [420] umass0:2:0: Attached to scbus2 Oct 28 12:27:09 r500 kernel: [420] da0 at umass-sim0 bus 0 scbus2 target 0 lun 0 Oct 28 12:27:09 r500 kernel: [420] da0: Removable Direct Access SCSI-2 device Oct 28 12:27:09 r500 kernel: [420] da0: Serial Number 22B39DCC Oct 28 12:27:09 r500 kernel: [420] da0: 40.000MB/s transfers Oct 28 12:27:09 r500 kernel: [420] da0: 7782MB (15937536 512 byte sectors) Oct 28 12:27:09 r500 kernel: [420] da0: quirks=0x2 Oct 28 12:27:09 r500 kernel: [421] GEOM_PART: integrity check failed (label/prot1, MBR) Oct 28 12:27:09 r500 kernel: [421] spa_sync: No writable vdev for prot1. Oct 28 12:27:10 r500 kernel: [422] spa_sync: No writable vdev for prot1. Oct 28 12:27:11 r500 kernel: [423] spa_sync: No writable vdev for prot1. Oct 28 12:27:12 r500 kernel: [423] GEOM_ELI: Device label/prot1.eli created. Oct 28 12:27:12 r500 kernel: [423] GEOM_ELI: Encryption: AES-XTS 256 Oct 28 12:27:12 r500 kernel: [423] GEOM_ELI: Crypto: software Oct 28 12:27:12 r500 ZFS: vdev state changed, pool_guid=2852801396507945726 vdev_guid=18325185749120546863 Oct 28 12:27:12 r500 kernel: [424] spa_sync: vdev for prot1 became writable again. Device loss was triggered and fixed with: sudo usbconfig -d 1.2 reset && zogftw import prot1 && sudo zpool clear prot1 This commit is experimental! It should be a nop unless the vfs.zfs.spa_sync_wait_for_writable_vdev sysctl is set, though. Setting the sysctl merely shrinks the race window and thus does not completely solve the problem. The problem can still be reproduced with an artifical test case like: Shell 1: mdconfig -u 0 -f /dpool/scratch/test-vdev.img zpool create test /dev/md0 while sleep 1; do mdconfig -d -u 0 -o force && mdconfig -f /dpool/scratch/test-vdev.img && zpool clear test; done Shell 2: # Cause writes to the pool from another shell, for example # by creating datasets. Log excerpt (from test begin to deadlock): Oct 29 12:34:28 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:44:42 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:45:04 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:43 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:43 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:44 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:44 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:45 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:45 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:46 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:46 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:47 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:47 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:48 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:48 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:49 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:49 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:51 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:51 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:52 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:52 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:53 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:54 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:54 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:55 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:55 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:56 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:56 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:57 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:57 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:59 kendra ZFS: vdev is removed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:46:59 kendra ZFS: vdev state changed, pool_guid=16039353738236808887 vdev_guid=3080051161477470469 Oct 29 12:47:00 kendra kernel: g_dev_taste: make_dev_p() failed (gp->name=md0, error=17) With the deadman enabled, this will also cause: panic: I/O to pool 'test' appears to be hung on vdev guid 3080051161477470469 at '/dev/md0'. cpuid = 0 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe01136af870 vpanic() at vpanic+0x182/frame 0xfffffe01136af8f0 panic() at panic+0x43/frame 0xfffffe01136af950 vdev_deadman() at vdev_deadman+0x127/frame 0xfffffe01136af9a0 vdev_deadman() at vdev_deadman+0x40/frame 0xfffffe01136af9f0 spa_deadman() at spa_deadman+0x86/frame 0xfffffe01136afa20 softclock_call_cc() at softclock_call_cc+0x1a3/frame 0xfffffe01136afaf0 softclock() at softclock+0x94/frame 0xfffffe01136afb20 intr_event_execute_handlers() at intr_event_execute_handlers+0x1b6/frame 0xfffffe01136afb60 ithread_loop() at ithread_loop+0xa6/frame 0xfffffe01136afbb0 fork_exit() at fork_exit+0x9c/frame 0xfffffe01136afbf0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01136afbf0 With test's txg_sync_thread being the offender: (kgdb) tid 101874 [Switching to thread 819 (Thread 101874)]#0 sched_switch (td=0xfffff800513649a0, newtd=, flags=) at /usr/src/sys/kern/sched_ule.c:1969 1969 cpuid = PCPU_GET(cpuid); (kgdb) where #0 sched_switch (td=0xfffff800513649a0, newtd=, flags=) at /usr/src/sys/kern/sched_ule.c:1969 #1 0xffffffff805a3a18 in mi_switch (flags=260, newtd=0x0) at /usr/src/sys/kern/kern_synch.c:470 #2 0xffffffff805ea15a in sleepq_wait (wchan=0x0, pri=0) at /usr/src/sys/kern/subr_sleepqueue.c:631 #3 0xffffffff80530509 in _cv_wait (cvp=0xfffff8002678ea98, lock=0xfffff8002678ea78) at /usr/src/sys/kern/kern_condvar.c:139 #4 0xffffffff81930bbb in zio_wait (zio=) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:1535 #5 0xffffffff818e4871 in dsl_pool_sync (dp=0xfffff80047dfd000, txg=76) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c:540 #6 0xffffffff81903653 in spa_sync (spa=0xfffff8009dfe2000, txg=76) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6734 #7 0xffffffff8190ccfa in txg_sync_thread (arg=0xfffff80047dfd000) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:517 #8 0xffffffff80556edc in fork_exit (callout=0xffffffff8190c970 , arg=0xfffff80047dfd000, frame=0xfffffe011c27bc00) at /usr/src/sys/kern/kern_fork.c:1011 #9 0xffffffff8085b91e in fork_trampoline () at /usr/src/sys/amd64/amd64/exception.S:609 #10 0x0000000000000000 in ?? () (kgdb) f 6 #6 0xffffffff81903653 in spa_sync (spa=0xfffff8009dfe2000, txg=76) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6734 (kgdb) p spa->spa_name $3 = 0xfffff8009dfe2000 "test" --- .../contrib/opensolaris/uts/common/fs/zfs/spa.c | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c index fff5f8fadbef..54cb224a7cdd 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c @@ -6817,6 +6817,31 @@ spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) rrw_exit(&dp->dp_config_rwlock, FTAG); } + +/* + * Check if there's at least one (potentionally) writable vdev. + */ +int +spa_has_writable_vdev(spa_t *spa) { + vdev_t *rvd = spa->spa_root_vdev; + vdev_t *vd; + int i; + + for (i = 0; i < rvd->vdev_children; i++) { + vd = rvd->vdev_child[i]; + if (!vd->vdev_cant_write) + return 1; + } + return 0; +} + +static int spa_sync_wait_for_writable_vdev = 0; +SYSCTL_INT(_vfs_zfs, OID_AUTO, spa_sync_wait_for_writable_vdev, CTLFLAG_RWTUN, + &spa_sync_wait_for_writable_vdev, 0, + "Let spa_sync() pause if no writable vdev is available. Experimental!"); + +static int spa_sync_deadlock_loops = 0; + /* * Sync the specified transaction group. New blocks may be dirtied as * part of the process, so we iterate until it converges. @@ -6950,6 +6975,29 @@ spa_sync(spa_t *spa, uint64_t txg) spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); spa_errlog_sync(spa, txg); + if (spa_sync_wait_for_writable_vdev) { + int waited_for_vdev = 0; + + while (!spa_has_writable_vdev(spa)) { + spa_config_exit(spa, SCL_CONFIG, FTAG); + + waited_for_vdev = 1; + printf("%s: No writable vdev for %s.\n", + __func__, spa_name(spa)); +#ifdef _KERNEL + pause("spa_sync", hz); +#else + sleep(1); +#endif + spa_sync_deadlock_loops++; + + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + } + if (waited_for_vdev) { + printf("%s: vdev for %s became writable again.\n", + __func__, spa_name(spa)); + } + } dsl_pool_sync(dp, txg); if (pass < zfs_sync_pass_deferred_free) { -- 2.11.0 From 0764538017d3904ab7decd0ccf07059d9f41b6ed Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 8 Jan 2016 13:30:21 +0100 Subject: [PATCH 223/257] ZFS ARC: Sync with FreeBSD bug #187594 (ZFS ARC behavior problem and fix) Main patch author: Karl Denninger PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=187594 PATCH URL: https://bz-attachments.freebsd.org/attachment.cgi?id=164051&action=diff&format=raw&headers=1 --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 60 +++++++++++++++++++++- .../contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 9 +++- .../contrib/opensolaris/uts/common/fs/zfs/zio.c | 3 ++ 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 55a1ca4c1ff3..1a55d5ceca01 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -366,6 +366,14 @@ int zfs_arc_shrink_shift = 0; int zfs_arc_p_min_shift = 0; uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */ u_int zfs_arc_free_target = 0; +u_int zfs_arc_wakeup_pager = 0; +u_int zfs_arc_wakeup_delay = 500; + +#define WAKE_PAGER +#ifdef WAKE_PAGER +#define WAKE_PAGER_CONSTANT 10 / 9 /* Pager wakeup threshold */ +static int arc_init_done = 0; /* We know arc_warm is valid */ +#endif /* WAKE_PAGER */ /* Absolute min for arc min / max is 16MB. */ static uint64_t arc_abs_min = 16 << 20; @@ -383,6 +391,9 @@ arc_free_target_init(void *unused __unused) { zfs_arc_free_target = vm_pageout_wakeup_thresh + ((vm_cnt.v_free_target - vm_pageout_wakeup_thresh) / 2); +#ifdef WAKE_PAGER + zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; +#endif /* WAKE_PAGER */ } SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, arc_free_target_init, NULL); @@ -406,6 +417,12 @@ SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RDTUN, SYSCTL_INT(_vfs_zfs, OID_AUTO, dynamic_write_buffer, CTLFLAG_RWTUN, &zfs_dynamic_write_buffer, 0, "Dynamically restrict dirty data when memory is low"); +#ifdef WAKE_PAGER +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_pager, CTLFLAG_RWTUN, + &zfs_arc_wakeup_pager, 0, "Wake VM below this number of pages"); +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_delay, CTLFLAG_RWTUN, + &zfs_arc_wakeup_delay, 0, "May wake up VM once this number of MS"); +#endif /* WAKE_PAGER */ /* * We don't have a tunable for arc_free_target due to the dependency on @@ -433,6 +450,9 @@ sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS) return (EINVAL); zfs_arc_free_target = val; +#ifdef WAKE_PAGER + zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; +#endif /* WAKE_PAGER */ return (0); } @@ -3866,6 +3886,11 @@ int64_t arc_pages_pp_reserve = 64; int64_t arc_swapfs_reserve = 64; /* + * Declare file-local static for event processor bypass + */ +static unsigned int arc_no_wake_event = 0; + +/* * Return the amount of memory that can be consumed before reclaim will be * needed. Positive if there is sufficient free memory, negative indicates * the amount of memory that needs to be freed up. @@ -3878,6 +3903,10 @@ arc_available_memory(void) free_memory_reason_t r = FMR_UNKNOWN; #ifdef _KERNEL +#ifdef WAKE_PAGER + sbintime_t now; + static sbintime_t last_pagedaemon_wake = 0; +#endif /* WAKE_PAGER */ if (needfree > 0) { n = PAGESIZE * (-needfree); if (n < lowest) { @@ -3896,6 +3925,26 @@ arc_available_memory(void) r = FMR_LOTSFREE; } +#ifdef WAKE_PAGER +/* + * If memory is less than the ARC wakeup threshold and time has expired since + * the last time we woke the pager... Do not execute until the ARC warms up. + */ + if ((arc_init_done) && + (((int64_t) freemem - zfs_arc_wakeup_pager) < 0) && + (arc_warm == B_TRUE) + ) { + now = getsbinuptime(); + if ((now - last_pagedaemon_wake) / SBT_1MS > zfs_arc_wakeup_delay) { + last_pagedaemon_wake = now; + arc_no_wake_event++; /* Set bypass flag for ARC */ + DTRACE_PROBE(arc__wake_pagedaemon); + pagedaemon_wakeup(); /* Wake the pager */ + } + } + +#endif /* WAKE_PAGER */ + #ifdef illumos /* * check that we're out of range of the pageout scanner. It starts to @@ -4004,6 +4053,7 @@ arc_available_memory(void) last_free_memory = lowest; last_free_reason = r; DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r); + return (lowest); } @@ -4034,9 +4084,10 @@ extern kmem_cache_t *range_seg_cache; * experimenting. */ static unsigned int arc_reaping_in_progress = 0; +static unsigned int arc_pagedaemon_ignore = 0; static sbintime_t last_reaping = 0; -static void __noinline +static __noinline void reap_arc_caches(void) { size_t i; @@ -5985,6 +6036,10 @@ static void arc_lowmem(void *arg __unused, int howto __unused) { + if (arc_no_wake_event) { /* Don't do it if we woke the pager */ + arc_no_wake_event = 0; /* Just clear the flag */ + return; + } mutex_enter(&arc_reclaim_lock); /* XXX: Memory deficit should be passed as argument. */ needfree = btoc(arc_c >> arc_shrink_shift); @@ -6299,6 +6354,9 @@ arc_init(void) printf(" in /boot/loader.conf.\n"); } #endif +#ifdef WAKE_PAGER + arc_init_done++; /* For anyone who wants to know */ +#endif /* WAKE_PAGER */ } void diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 6e89c7d3b6a5..691a651c8ef7 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1123,8 +1123,13 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) ASSERT3U(dirty, <, zfs_dirty_data_max_internal); now = gethrtime(); - min_tx_time = zfs_delay_scale * - (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); + if (dirty >= zfs_dirty_data_max_internal) {/* No scaling if overcommitted */ + min_tx_time = zfs_delay_scale * + (dirty - delay_min_bytes); + } else { + min_tx_time = zfs_delay_scale * + (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); + } if (now > tx->tx_start + min_tx_time) return; diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c index d5ab54271619..41664263f91a 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c @@ -45,6 +45,9 @@ SYSCTL_DECL(_vfs_zfs); SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); +/* KD 2015-07-15 Change class to "int" from "static int" as we reference + * this as an extern elsewhere + */ #if defined(__amd64__) int zio_use_uma = 1; #else -- 2.11.0 From eabcc43289003ced75f468a1779365f3dabfdb08 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 17:07:24 +0200 Subject: [PATCH 224/257] sys/cddl: Allow to modify the ZFS deadman sysctls after the system is up There does not seem to be a technical reason why it shouldn't be done and being able to temporarily disable the deadman is useful when importing a pool that is backed by remote storage (such as ggated running as onion service). Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c index c247388468a6..c3fc2c979037 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c @@ -357,13 +357,13 @@ SYSCTL_PROC(_vfs_zfs, OID_AUTO, debug_flags, CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RWTUN, 0, sizeof(int), sysctl_vfs_zfs_debug_flags, "IU", "Debug flags for ZFS testing."); -SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_synctime_ms, CTLFLAG_RDTUN, +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_synctime_ms, CTLFLAG_RWTUN, &zfs_deadman_synctime_ms, 0, "Stalled ZFS I/O expiration time in milliseconds"); -SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_checktime_ms, CTLFLAG_RDTUN, +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_checktime_ms, CTLFLAG_RWTUN, &zfs_deadman_checktime_ms, 0, "Period of checks for stalled ZFS I/O in milliseconds"); -SYSCTL_INT(_vfs_zfs, OID_AUTO, deadman_enabled, CTLFLAG_RDTUN, +SYSCTL_INT(_vfs_zfs, OID_AUTO, deadman_enabled, CTLFLAG_RWTUN, &zfs_deadman_enabled, 0, "Kernel panic on stalled ZFS I/O"); SYSCTL_INT(_vfs_zfs, OID_AUTO, spa_asize_inflation, CTLFLAG_RWTUN, &spa_asize_inflation, 0, "Worst case inflation factor for single sector writes"); -- 2.11.0 From 3c737dbe53e4967132532faf8bf7e00f834bbf28 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 Jan 2016 13:32:24 +0100 Subject: [PATCH 225/257] ZFS ARC: If there's less than 1 GB of RAM, set the ARC max to 50% of it Now the code matches the comment again. The limit was increased to 6/8 in r172130 and reduced to 5/8 in r175633 again to mitigate 'kmem_map too small panics'. As it turns out, 5/8 is still too much and can result in deadlocks caused by the vm system running out of pages: (kgdb) p vm_cnt $7 = {v_swtch = 0, v_trap = 0, v_syscall = 0, v_intr = 0, v_soft = 0, v_vm_faults = 0, v_io_faults = 0, v_cow_faults = 0, v_cow_optim = 0, v_zfod = 0, v_ozfod = 0, v_swapin = 0, v_swapout = 0, v_swappgsin = 0, v_swappgsout = 0, v_vnodein = 0, v_vnodeout = 0, v_vnodepgsin = 0, v_vnodepgsout = 0, v_intrans = 0, v_reactivated = 0, v_pdwakeups = 878, v_pdpages = 0, v_tcached = 0, v_dfree = 0, v_pfree = 0, v_tfree = 0, v_page_size = 4096, v_page_count = 247933, v_free_reserved = 372, v_free_target = 5324, v_free_min = 1610, v_free_count = 2, v_wire_count = 72901, v_active_count = 174634, v_inactive_target = 7986, v_inactive_count = 395, v_cache_count = 0, v_pageout_free_min = 34, v_interrupt_free_min = 2, v_free_severe = 991, v_forks = 0, v_vforks = 0, v_rforks = 0, v_kthreads = 0, v_forkpages = 0, v_vforkpages = 0, v_rforkpages = 0, v_kthreadpages = 0, v_spare = 0xffffffff8141770c} Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 1a55d5ceca01..5e80d2c93439 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -6203,7 +6203,7 @@ arc_init(void) arc_c_max = (arc_c * 8) - (1 << 30); else arc_c_max = arc_c_min; - arc_c_max = MAX(arc_c * 5, arc_c_max); + arc_c_max = MAX(arc_c * 4, arc_c_max); /* * In userland, there's only the memory pressure that we artificially -- 2.11.0 From bc07bf6206b7e6f96dcfc821f6c639a5e80b435a Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 26 Jan 2016 12:44:01 +0100 Subject: [PATCH 226/257] ZFS ARC: Remove the 'WAKE_PAGER' ... as I suspect that the "Don't do it if we woke the pager" code in arc_lowmem() increase the chances that the vm runs out of free pages. It's also not obvious (to me) that the WAKE_PAGER does anything useful. Quoting https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=187594#c244: "I have not yet tried Karl's latest patch but like to point out that some of the underlying assumptions about how the vm pager behaves seem incorrect to me. For example I'd expect calling pagedaemon_wakeup() without memory pressure (from the pagers point of view) to be pretty close to a nop as vm_pageout_worker() does its own checks before doing any heavy lifting: http://fxr.watson.org/fxr/source/vm/vm_pageout.c#L1634 Also note that vm_pageout_worker() is already called at least once per second anyway: [fk@polizei-erziehung ~]$ sudo /usr/src/share/dtrace/monitor-page-scanner 2015 Nov 1 17:20:45: Monitoring the page scanner. Minimum pass value to show 'boring' scans without memory pressure or inactive page surplus: 2 (Launder dirty pages). Press CTRL-C to abort. 2015 Nov 1 17:21:45: Scan goals in the previous minute: Update active LRU/deactivate pages 60 2015 Nov 1 17:22:45: Scan goals in the previous minute: Update active LRU/deactivate pages 60 I'm not claiming that increasing the frequency when there's no memory pressure causes any harm (besides code complexity), but I'm not convinced that it has the intended effect and needs to be triggered from ZFS (as opposed to changing the pager defaults)." Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 45 ---------------------- 1 file changed, 45 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 5e80d2c93439..9c844e30e9e5 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -369,12 +369,6 @@ u_int zfs_arc_free_target = 0; u_int zfs_arc_wakeup_pager = 0; u_int zfs_arc_wakeup_delay = 500; -#define WAKE_PAGER -#ifdef WAKE_PAGER -#define WAKE_PAGER_CONSTANT 10 / 9 /* Pager wakeup threshold */ -static int arc_init_done = 0; /* We know arc_warm is valid */ -#endif /* WAKE_PAGER */ - /* Absolute min for arc min / max is 16MB. */ static uint64_t arc_abs_min = 16 << 20; @@ -391,9 +385,6 @@ arc_free_target_init(void *unused __unused) { zfs_arc_free_target = vm_pageout_wakeup_thresh + ((vm_cnt.v_free_target - vm_pageout_wakeup_thresh) / 2); -#ifdef WAKE_PAGER - zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; -#endif /* WAKE_PAGER */ } SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, arc_free_target_init, NULL); @@ -417,12 +408,6 @@ SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RDTUN, SYSCTL_INT(_vfs_zfs, OID_AUTO, dynamic_write_buffer, CTLFLAG_RWTUN, &zfs_dynamic_write_buffer, 0, "Dynamically restrict dirty data when memory is low"); -#ifdef WAKE_PAGER -SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_pager, CTLFLAG_RWTUN, - &zfs_arc_wakeup_pager, 0, "Wake VM below this number of pages"); -SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_delay, CTLFLAG_RWTUN, - &zfs_arc_wakeup_delay, 0, "May wake up VM once this number of MS"); -#endif /* WAKE_PAGER */ /* * We don't have a tunable for arc_free_target due to the dependency on @@ -450,9 +435,6 @@ sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS) return (EINVAL); zfs_arc_free_target = val; -#ifdef WAKE_PAGER - zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; -#endif /* WAKE_PAGER */ return (0); } @@ -3903,10 +3885,6 @@ arc_available_memory(void) free_memory_reason_t r = FMR_UNKNOWN; #ifdef _KERNEL -#ifdef WAKE_PAGER - sbintime_t now; - static sbintime_t last_pagedaemon_wake = 0; -#endif /* WAKE_PAGER */ if (needfree > 0) { n = PAGESIZE * (-needfree); if (n < lowest) { @@ -3925,26 +3903,6 @@ arc_available_memory(void) r = FMR_LOTSFREE; } -#ifdef WAKE_PAGER -/* - * If memory is less than the ARC wakeup threshold and time has expired since - * the last time we woke the pager... Do not execute until the ARC warms up. - */ - if ((arc_init_done) && - (((int64_t) freemem - zfs_arc_wakeup_pager) < 0) && - (arc_warm == B_TRUE) - ) { - now = getsbinuptime(); - if ((now - last_pagedaemon_wake) / SBT_1MS > zfs_arc_wakeup_delay) { - last_pagedaemon_wake = now; - arc_no_wake_event++; /* Set bypass flag for ARC */ - DTRACE_PROBE(arc__wake_pagedaemon); - pagedaemon_wakeup(); /* Wake the pager */ - } - } - -#endif /* WAKE_PAGER */ - #ifdef illumos /* * check that we're out of range of the pageout scanner. It starts to @@ -6354,9 +6312,6 @@ arc_init(void) printf(" in /boot/loader.conf.\n"); } #endif -#ifdef WAKE_PAGER - arc_init_done++; /* For anyone who wants to know */ -#endif /* WAKE_PAGER */ } void -- 2.11.0 From 03b9335717d3c51b533d7da90598106a474deba8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 2 May 2016 12:05:41 +0200 Subject: [PATCH 227/257] ZFS: Fix a typo in the delay_min_dirty_percent sysctl description Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c index 33820f76e7b3..bcf9b055f96c 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c @@ -165,7 +165,7 @@ static int sysctl_zfs_delay_min_dirty_percent(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vfs_zfs, OID_AUTO, delay_min_dirty_percent, CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(int), sysctl_zfs_delay_min_dirty_percent, "I", - "The limit of outstanding dirty data before transations are delayed"); + "The limit of outstanding dirty data before transactions are delayed"); static int sysctl_zfs_delay_scale(SYSCTL_HANDLER_ARGS); /* No zfs_delay_scale tunable due to limit requirements */ -- 2.11.0 From f30c5a953d60af1850a2e625adcdcdc0d8580e38 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 30 Apr 2016 09:41:25 +0200 Subject: [PATCH 228/257] ZFS: Make dmu_tx_delay() immune to zfs_dirty_data_max_internal changes while it's running as it can result in underflows and possible deadlocks Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 691a651c8ef7..bc006edc0f95 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1107,10 +1107,11 @@ static void dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) { dsl_pool_t *dp = tx->tx_pool; - uint64_t delay_min_bytes = - zfs_dirty_data_max_internal * zfs_delay_min_dirty_percent / 100; + uint64_t delay_min_bytes; hrtime_t wakeup, min_tx_time, now; + uint64_t dirty_data_max = zfs_dirty_data_max_internal; + delay_min_bytes = dirty_data_max * zfs_delay_min_dirty_percent / 100; if (dirty <= delay_min_bytes) return; @@ -1120,15 +1121,15 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) * have to handle the case of it being >= the max, which could * cause a divide-by-zero if it's == the max. */ - ASSERT3U(dirty, <, zfs_dirty_data_max_internal); + ASSERT3U(dirty, <, dirty_data_max); now = gethrtime(); - if (dirty >= zfs_dirty_data_max_internal) {/* No scaling if overcommitted */ + if (dirty >= dirty_data_max) {/* No scaling if overcommitted */ min_tx_time = zfs_delay_scale * (dirty - delay_min_bytes); } else { min_tx_time = zfs_delay_scale * - (dirty - delay_min_bytes) / (zfs_dirty_data_max_internal - dirty); + (dirty - delay_min_bytes) / (dirty_data_max - dirty); } if (now > tx->tx_start + min_tx_time) return; -- 2.11.0 From 37efc2894d0052cf306eb5123d71972baef372a3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 5 May 2016 13:59:47 +0200 Subject: [PATCH 229/257] share/skel: Detach skelleton files for csh and rsh from the build Obtained from: ElectroBSD --- share/skel/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/skel/Makefile b/share/skel/Makefile index 6b67d2bdbec5..8c51c43c3439 100644 --- a/share/skel/Makefile +++ b/share/skel/Makefile @@ -2,9 +2,9 @@ # $FreeBSD$ FILESGROUPS= FILES1 FILES2 -FILES1= dot.cshrc dot.login dot.login_conf dot.mailrc dot.profile \ +FILES1= dot.login_conf dot.mailrc dot.profile \ dot.shrc -FILES2= dot.mail_aliases dot.rhosts +FILES2= dot.mail_aliases FILES1DIR= ${SHAREDIR}/skel FILES2DIR= ${SHAREDIR}/skel FILES1MODE= 0644 -- 2.11.0 From fb14661532fcbe89327cad6b5b1d24c7386f211f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 27 Apr 2016 10:48:23 +0200 Subject: [PATCH 230/257] ZFS: Make the dynamic write buffer more effective for memory-constrained systems - Remove the previously hardcoded 256 MB lower limit which is far too much for systems with 2 GB of RAM or less. - Prevent an integer underflow if the free count is below the free target. The vm seems to treat vm_cnt.v_free_target more like a guide line than a rule, and it's unsafe to assume that it will always be below the vm_cnt.v_free_count. (kgdb) p zfs_dirty_data_max_internal $8 = 10000000 (kgdb) p vm_cnt.v_free_count - vm_cnt.v_free_target $9 = 18446744073709546294 (kgdb) p vm_cnt.v_free_count $10 = 2 (kgdb) p vm_cnt.v_free_target $11 = 5324 If the free count is below the target, use half the free count as limit instead. There is probably a more appropriate limit, but for now this has to do. Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index bc006edc0f95..f1a0688cb1b3 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1350,16 +1350,16 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) #ifdef _KERNEL /* - * KD 2014-09-22 * If UMA is enabled it can only return a previously-used block - * of identical size to what it had out before. If it's not the + * of identical size to what it had out before. If it's not the * same size it will allocate a new one. This is a problem because * dirty_data_max is the total dirty write data allowed out at any * given time, but with UMA on that can multiply by the number of * different block sizes (!!) requested in terms of free RAM that * is left allocated but unused. For this reason never allow * dirty_data_max to exceed the difference between the paging - * threshold and the current free memory, with a minimum of 256MB. + * threshold and the current free memory. + * * This throttles "burst" allocations and prevents the system from * choking during times of high write I/O demand. * @@ -1368,13 +1368,17 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) * * Note that we work on the zfs_dirty_data_max_internal variable, * because the user may set zfs_dirty_data_max himself and we must - * must honor that as a hard cap so it remains a usable tunable value. + * honor that as a hard cap so it remains a usable tunable value. */ if (zio_use_uma && zfs_dynamic_write_buffer) { - zfs_dirty_data_max_internal = 1 << 28; - zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); - zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max); - zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, zfs_dirty_data_max_max); + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max, zfs_dirty_data_max_max); + if (vm_cnt.v_free_count > vm_cnt.v_free_target) { + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, + ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); + } else { + zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, + ptob(vm_cnt.v_free_count / 2)); + } if (last_max != (zfs_dirty_data_max_internal / (1024 * 1024))) { last_max = zfs_dirty_data_max_internal / (1024 * 1024); DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); -- 2.11.0 From 2aa8fde998bca6cbdafa2c2c6b62d3443ad7fb12 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 27 Apr 2016 11:21:17 +0200 Subject: [PATCH 231/257] ZFS: Add sysctl to show how often the write buffer was limitted Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index f1a0688cb1b3..6cc268812cde 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -46,6 +46,12 @@ typedef void (*dmu_tx_hold_func_t)(dmu_tx_t *tx, struct dnode *dn, extern int zio_use_uma; /* Needs to be visible; DO NOT MODIFY! */ int zfs_dynamic_write_buffer = 1; /* Dynamically tune writes */ +SYSCTL_DECL(_vfs_zfs); +static uint64_t zfs_dynamic_write_buffer_hits = 0; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dynamic_write_buffer_hits, CTLFLAG_RWTUN, + &zfs_dynamic_write_buffer_hits, 0, + "Number of times the dynamic write buffer was limitted"); + dmu_tx_t * dmu_tx_create_dd(dsl_dir_t *dd) { @@ -1383,6 +1389,9 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) last_max = zfs_dirty_data_max_internal / (1024 * 1024); DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); } + if (zfs_dirty_data_max_internal < zfs_dirty_data_max) { + zfs_dynamic_write_buffer_hits++; + } } else { zfs_dirty_data_max_internal = zfs_dirty_data_max; } -- 2.11.0 From c653f8b2689907282055baebd07f3ae7e42e0bf1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 27 Apr 2016 11:52:35 +0200 Subject: [PATCH 232/257] ZFS: Add sysctl to read the dirty_data_max_internal Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c index bcf9b055f96c..d99323fd21a3 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c @@ -150,6 +150,10 @@ SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dirty_data_max_max, CTLFLAG_RDTUN, &zfs_dirty_data_max_max, 0, "The absolute cap on dirty_data_max when auto calculating"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dirty_data_max_internal, CTLFLAG_RD, + &zfs_dirty_data_max_internal, 0, + "The currently active limit of dirty data (may dynamically change)"); + static int sysctl_zfs_dirty_data_max_percent(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vfs_zfs, OID_AUTO, dirty_data_max_percent, CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RWTUN, 0, sizeof(int), -- 2.11.0 From 945275591d4027b3d176d67f7aca89bd98311036 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 28 Apr 2016 13:52:18 +0200 Subject: [PATCH 233/257] ZFS: Add a sysctl to access the lowest write buffer limit Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 6cc268812cde..51ded24b981f 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -52,6 +52,11 @@ SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dynamic_write_buffer_hits, CTLFLAG_RWTUN, &zfs_dynamic_write_buffer_hits, 0, "Number of times the dynamic write buffer was limitted"); +static uint64_t zfs_lowest_dynamic_write_buffer_limit = 0; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, lowest_dynamic_write_buffer_limit, CTLFLAG_RD, + &zfs_lowest_dynamic_write_buffer_limit, 0, + "The lowest value the dynamic write buffer was limitted to"); + dmu_tx_t * dmu_tx_create_dd(dsl_dir_t *dd) { @@ -1392,6 +1397,10 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) if (zfs_dirty_data_max_internal < zfs_dirty_data_max) { zfs_dynamic_write_buffer_hits++; } + if (zfs_lowest_dynamic_write_buffer_limit == 0 || + zfs_lowest_dynamic_write_buffer_limit >= zfs_dirty_data_max_internal) { + zfs_lowest_dynamic_write_buffer_limit = zfs_dirty_data_max_internal; + } } else { zfs_dirty_data_max_internal = zfs_dirty_data_max; } -- 2.11.0 From 65880df0d8ad74b7c372c9754bd399adc369d8b1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 11 May 2016 12:41:12 +0200 Subject: [PATCH 234/257] ZFS: Attempt to let the spa deadman store the last and the maximum vdev sync delay As it's useful to know when tuning the timeout. XXX: This looks like it should work, but actually doesn't. Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c index 2d8d93fc8e76..3566c2b45f63 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c @@ -91,6 +91,16 @@ SYSCTL_NODE(_vfs_zfs, OID_AUTO, vdev, CTLFLAG_RW, 0, "ZFS VDEV"); static uint64_t zfs_max_auto_ashift = SPA_MAXASHIFT; static uint64_t zfs_min_auto_ashift = SPA_MINASHIFT; +static uint64_t zfs_max_vdev_sync_delay = 0; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, vdev_sync_delay_max, CTLFLAG_RWTUN, + &zfs_max_vdev_sync_delay, 0, + "Maximum vdev sync delay seen by the spa deadman"); + +static uint64_t zfs_last_vdev_sync_delay = 0; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, vdev_sync_delay_last, CTLFLAG_RWTUN, + &zfs_last_vdev_sync_delay, 0, + "Last vdev sync delay seen seen by the spa deadman"); + static int sysctl_vfs_zfs_max_auto_ashift(SYSCTL_HANDLER_ARGS) { @@ -3616,6 +3626,9 @@ vdev_deadman(vdev_t *vd) */ fio = avl_first(&vq->vq_active_tree); delta = gethrtime() - fio->io_timestamp; + zfs_last_vdev_sync_delay = delta; + if (zfs_max_vdev_sync_delay < delta) + zfs_max_vdev_sync_delay = delta; if (delta > spa_deadman_synctime(spa)) { zfs_dbgmsg("SLOW IO: zio timestamp %lluns, " "delta %lluns, last io %lluns", -- 2.11.0 From 169f8544c6647dfae0a564c3891c1deca489a690 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 31 May 2016 23:46:19 +0200 Subject: [PATCH 235/257] ZFS: Use https for the error message URLs Obtained from: ElectroBSD --- cddl/contrib/opensolaris/cmd/zpool/zpool_main.c | 6 +++--- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c index e2dc24c5fa91..1ed99c9b0c22 100644 --- a/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c +++ b/cddl/contrib/opensolaris/cmd/zpool/zpool_main.c @@ -1826,7 +1826,7 @@ show_import(nvlist_t *config) } if (msgid != NULL) - (void) printf(gettext(" see: http://illumos.org/msg/%s\n"), + (void) printf(gettext(" see: https://illumos.org/msg/%s\n"), msgid); (void) printf(gettext(" config:\n\n")); @@ -4124,7 +4124,7 @@ print_dedup_stats(nvlist_t *config) * pool: tank * status: DEGRADED * reason: One or more devices ... - * see: http://illumos.org/msg/ZFS-xxxx-01 + * see: https://illumos.org/msg/ZFS-xxxx-01 * config: * mirror DEGRADED * c1t0d0 OK @@ -4381,7 +4381,7 @@ status_callback(zpool_handle_t *zhp, void *data) } if (msgid != NULL) - (void) printf(gettext(" see: http://illumos.org/msg/%s\n"), + (void) printf(gettext(" see: https://illumos.org/msg/%s\n"), msgid); if (config != NULL) { diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c index 54cb224a7cdd..0ef9065de044 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c @@ -2631,7 +2631,7 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config, cmn_err(CE_WARN, "pool '%s' could not be " "loaded as it was last accessed by " "another system (host: %s hostid: 0x%lx). " - "See: http://illumos.org/msg/ZFS-8000-EY", + "See: https://illumos.org/msg/ZFS-8000-EY", spa_name(spa), hostname, (unsigned long)hostid); return (SET_ERROR(EBADF)); -- 2.11.0 From 2505cc72154b11223bf227b5e94057a2c7425b13 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 29 Apr 2016 19:38:53 +0200 Subject: [PATCH 236/257] ZFS: Add a (tunable) lower limit for the dynamic write buffer Default to using 0.5% of the system's physical memory. Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 13 +++++++++++++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 1 + sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c | 5 +++++ .../contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h | 1 + 4 files changed, 20 insertions(+) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 9c844e30e9e5..61b1bcdffef2 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -6276,6 +6276,19 @@ arc_init(void) zfs_dirty_data_max = MIN(zfs_dirty_data_max, zfs_dirty_data_max_max); } + /* + * If memory is tight, reduce the dynamic write buffer + * as low as 0.5% of the physical memory. This allows + * a system with 1 GB of physical memory to remain stable + * while building kernels in a loop for days. + * Using 1% (10 MB) proved to be too much. + * + * Systems with 2 GB of physical memory don't seem to + * reach the lower limit while building kernels. + */ + if (zfs_dirty_data_max_min == 0) { + zfs_dirty_data_max_min = ptob(physmem) / 200; + } #ifdef _KERNEL if (TUNABLE_INT_FETCH("vfs.zfs.prefetch_disable", &zfs_prefetch_disable)) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 51ded24b981f..7c1e3a6df36a 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1394,6 +1394,7 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) last_max = zfs_dirty_data_max_internal / (1024 * 1024); DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); } + zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, zfs_dirty_data_max_min); if (zfs_dirty_data_max_internal < zfs_dirty_data_max) { zfs_dynamic_write_buffer_hits++; } diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c index d99323fd21a3..b8d742844e37 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c @@ -105,6 +105,7 @@ uint64_t zfs_dirty_data_max; uint64_t zfs_dirty_data_max_internal; uint64_t zfs_dirty_data_max_max = 4ULL * 1024 * 1024 * 1024; +uint64_t zfs_dirty_data_max_min = 0; int zfs_dirty_data_max_percent = 10; /* @@ -154,6 +155,10 @@ SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dirty_data_max_internal, CTLFLAG_RD, &zfs_dirty_data_max_internal, 0, "The currently active limit of dirty data (may dynamically change)"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, dirty_data_max_min, CTLFLAG_RWTUN, + &zfs_dirty_data_max_min, 0, + "The absolute lower cap on dirty_data_max_internal when auto calculating"); + static int sysctl_zfs_dirty_data_max_percent(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vfs_zfs, OID_AUTO, dirty_data_max_percent, CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RWTUN, 0, sizeof(int), diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h index ee97b5719cd2..c0444da17104 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h @@ -52,6 +52,7 @@ struct dsl_scan; extern uint64_t zfs_dirty_data_max; extern uint64_t zfs_dirty_data_max_internal; extern uint64_t zfs_dirty_data_max_max; +extern uint64_t zfs_dirty_data_max_min; extern uint64_t zfs_dirty_data_sync; extern int zfs_dirty_data_max_percent; extern int zfs_delay_min_dirty_percent; -- 2.11.0 From 34aaa9a0fc127ec321a1ca8cfcc1f1ece6983d4b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 3 May 2016 16:13:29 +0200 Subject: [PATCH 237/257] dmu_tx_assign(): Limit zfs_dirty_data_max_internal growth ... to steps of zfs_dirty_data_max_min or less. As dmu_tx_assign() is called quite frequently the growth should probably additionally be delayed based on time, though. Obtained from: ElectroBSD --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 7c1e3a6df36a..6342d397928a 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1346,7 +1346,7 @@ int dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) { int err; - static uint64_t last_max; + static uint64_t last_max = 4ULL * 1024 * 1024 * 1024; ASSERT(tx->tx_txg == 0); ASSERT(txg_how == TXG_WAIT || txg_how == TXG_NOWAIT || @@ -1390,9 +1390,10 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, ptob(vm_cnt.v_free_count / 2)); } - if (last_max != (zfs_dirty_data_max_internal / (1024 * 1024))) { - last_max = zfs_dirty_data_max_internal / (1024 * 1024); - DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max); + zfs_dirty_data_max_internal = MIN(last_max + zfs_dirty_data_max_min, zfs_dirty_data_max_internal); + if (last_max != zfs_dirty_data_max_internal) { + last_max = zfs_dirty_data_max_internal; + DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max / (1024 * 1024)); } zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, zfs_dirty_data_max_min); if (zfs_dirty_data_max_internal < zfs_dirty_data_max) { -- 2.11.0 From 98e570fb4658252aa06716df381f92b09af2560e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 8 May 2016 14:13:41 +0200 Subject: [PATCH 238/257] dmu_tx_assign(): Use local variable and use the whole v_free_count if it's below the target Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c index 6342d397928a..0c10b307698b 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c @@ -1382,27 +1382,29 @@ dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) * honor that as a hard cap so it remains a usable tunable value. */ if (zio_use_uma && zfs_dynamic_write_buffer) { - zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max, zfs_dirty_data_max_max); + uint64_t new_max; + + new_max = MIN(zfs_dirty_data_max, zfs_dirty_data_max_max); if (vm_cnt.v_free_count > vm_cnt.v_free_target) { - zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, + new_max = MIN(new_max, ptob(vm_cnt.v_free_count - vm_cnt.v_free_target)); } else { - zfs_dirty_data_max_internal = MIN(zfs_dirty_data_max_internal, - ptob(vm_cnt.v_free_count / 2)); + new_max = MIN(new_max, vm_cnt.v_free_count); } - zfs_dirty_data_max_internal = MIN(last_max + zfs_dirty_data_max_min, zfs_dirty_data_max_internal); - if (last_max != zfs_dirty_data_max_internal) { - last_max = zfs_dirty_data_max_internal; + new_max = MIN(last_max + zfs_dirty_data_max_min, new_max); + if (last_max != new_max) { + last_max = new_max; DTRACE_PROBE1(dmu__tx_dirty, uint64_t, last_max / (1024 * 1024)); } - zfs_dirty_data_max_internal = MAX(zfs_dirty_data_max_internal, zfs_dirty_data_max_min); - if (zfs_dirty_data_max_internal < zfs_dirty_data_max) { + new_max = MAX(new_max, zfs_dirty_data_max_min); + if (new_max < zfs_dirty_data_max) { zfs_dynamic_write_buffer_hits++; } if (zfs_lowest_dynamic_write_buffer_limit == 0 || - zfs_lowest_dynamic_write_buffer_limit >= zfs_dirty_data_max_internal) { - zfs_lowest_dynamic_write_buffer_limit = zfs_dirty_data_max_internal; + zfs_lowest_dynamic_write_buffer_limit >= new_max) { + zfs_lowest_dynamic_write_buffer_limit = new_max; } + zfs_dirty_data_max_internal = new_max; } else { zfs_dirty_data_max_internal = zfs_dirty_data_max; } -- 2.11.0 From 1a64e7f4993553eb6a691daf4b65e42db26e56dc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 22 Sep 2015 16:05:49 +0200 Subject: [PATCH 239/257] sys/vm: Limit the inactive pages more aggressively Currently the ZFS ARC does not take the inactive pages into account when calculating its target size. If there's no limit for the inactive pages, the ARC may shrink to its own limit while the number of inactive pages continues to grow: last pid: 28429; load averages: 0.48, 0.46, 0.41 up 0+03:39:07 17:24:59 91 processes: 2 running, 88 sleeping, 1 waiting CPU: 1.4% user, 0.0% nice, 12.7% system, 0.2% interrupt, 85.7% idle Mem: 396M Active, 489M Inact, 986M Wired, 292K Cache, 5202K Buf, 43M Free ARC: 351M Total, 90M MFU, 44M MRU, 6839K Anon, 7810K Header, 203M Other, 350M Target Swap: 2048M Total, 99M Used, 1949M Free, 4% Inuse PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU COMMAND 11 root 2 155 ki31 0K 32K RUN 0 377:37 170.34% idle 26625 fk 17 36 0 175M 24504K uwait 1 0:09 8.40% git 0 root 468 -16 0 0K 7488K swapin 1 3:29 6.26% kernel 22 root 1 20 - 0K 16K geli:w 1 4:16 5.06% g_eli[1] ada0s1d [...] 2015 Sep 21 17:24:58: Scan goals in the previous minute: Update active LRU/deactivate pages 60 2015 Sep 21 17:24:58: Seconds since last 'Move inactive to cache or free' pass: 1477 2015 Sep 21 17:24:58: Seconds since last 'Launder dirty pages' pass: 9273 With this commit, the system lets the ARC indirectly put pressure on the inactive pages until a given target is reached. A couple of sysctls can be used to set various limits, the auto-tuned default should work reasonably well, though. Note that suboptimal tuning can result in excessive paging. Screenshot (made with previous version of this commit): https://www.fabiankeil.de/bilder/electrobsd/kernel-compilation-with-inactive-page-limit-enabled.png XXX: After rebasing on r300865 this commit caused a bunch of conflicts that may not have been addressed ideally. Obtained from: ElectroBSD --- sys/vm/vm_pageout.c | 194 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 157 insertions(+), 37 deletions(-) diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 3e36f7c1c3fa..141c087becfa 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -140,15 +140,17 @@ SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start, SDT_PROVIDER_DEFINE(vm); SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan); -SDT_PROBE_DEFINE4(vm, , , before__inactive__scan, "struct vm_domain *vmd", - "int pass", "int page_shortage", "int deficit"); -SDT_PROBE_DEFINE5(vm, , , after__inactive__scan, "struct vm_domain *vmd", +SDT_PROBE_DEFINE5(vm, , , before__inactive__scan, "struct vm_domain *vmd", + "int pass", "int page_shortage", "int deficit", "int inactive_page_surplus"); +SDT_PROBE_DEFINE6(vm, , , after__inactive__scan, "struct vm_domain *vmd", "int pass", "int page_shortage", "int addl_page_shortage", - "int vnodes_skipped"); + "int vnodes_skipped", "int inactive_page_surplus"); SDT_PROBE_DEFINE3(vm, , , before__active__scan, "struct vm_domain *vmd", "int pass", "int page_shortage"); SDT_PROBE_DEFINE3(vm, , , after__active__scan, "struct vm_domain *vmd", "int pass", "int page_shortage"); +SDT_PROBE_DEFINE3(vm, , , checked__inactive__pages, "int pages_to_free", + "int pages_above_limit", "int enforced_limit"); #if !defined(NO_SWAPPING) /* the kernel process "vm_daemon"*/ @@ -230,6 +232,36 @@ SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts, SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts, CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages"); +static int inactive_page_limit_enabled = 1; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_enabled, CTLFLAG_RW, + &inactive_page_limit_enabled, 0, + "Free inactive pages above the target more aggressively. " + "Values: 0 (disabled), 1 (soft mode, only apply limit if free " + "page count is low), 2 (ignore free count)"); + +static int inactive_page_limit_offset_i; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_offset_i, CTLFLAG_RW, + &inactive_page_limit_offset_i, 0, + "Number of inactive pages relative to the inactive target " + "required for inactive pages to be freed."); + +static int inactive_page_limit_offset_f; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_offset_f, CTLFLAG_RW, + &inactive_page_limit_offset_f, 0, + "Number of free pages relative to the free target required for " + "the inactive memory limit to be applied."); + +static int inactive_pages_to_free_max = 1000; +SYSCTL_INT(_vm, OID_AUTO, inactive_pages_to_free_max, CTLFLAG_RW, + &inactive_pages_to_free_max, 0, + "Maximum number of inactive pages above the target to free at once."); + +static int inactive_page_limit_threshold = 1000; +SYSCTL_INT(_vm, OID_AUTO, inactive_page_limit_threshold, CTLFLAG_RW, + &inactive_page_limit_threshold, 0, + "Number of inactive pages above the limit required " + "to trigger a inactive page reduction."); + static int pageout_lock_miss; SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss, CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout"); @@ -848,6 +880,41 @@ unlock_mp: return (error); } +static int +vm_pageout_get_inactive_page_surplus(void) +{ + int pages_to_free; + int pages_above_limit; + int enforced_limit; + + /* Return early so the DTrace probe does not fire. */ + if (!inactive_page_limit_enabled) + return (0); + + enforced_limit = vm_cnt.v_inactive_target + inactive_page_limit_offset_i; + pages_above_limit = vm_cnt.v_inactive_count - enforced_limit; + + /* + * We want to free inactive pages if the threshold of inactive + * pages above the limit is reached and we are either using + * a hard limit, or the number of free pages is below the + * free page limit. + */ + if ((pages_above_limit >= inactive_page_limit_threshold) && + ((inactive_page_limit_enabled == 2) || + (vm_paging_target() + inactive_page_limit_offset_f > 0))) { + pages_to_free = imin(inactive_pages_to_free_max, + pages_above_limit); + } else { + pages_to_free = 0; + } + + SDT_PROBE3(vm, , , checked__inactive__pages, pages_to_free, + pages_above_limit, enforced_limit); + + return (pages_to_free); +} + #define VMD_PASS_MAX 3 /* * vm_pageout_scan does the dirty work for the pageout daemon. @@ -870,15 +937,35 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) int maxlaunder, maxscan, page_shortage, scan_tick, scanned; int starting_page_shortage, vnodes_skipped; boolean_t pageout_ok, queue_locked; + int inactive_page_surplus; KASSERT(pass <= VMD_PASS_MAX, ("vm_pageout_scan: Invalid pass code %d", pass)); /* + * The addl_page_shortage is the number of temporarily + * stuck pages in the inactive queue. In other words, the + * number of pages from the inactive count that should be + * discounted in setting the target for the active queue scan. + */ + addl_page_shortage = 0; + + /* + * Calculate the number of pages that we want to free. + */ + if (pass > 0) { + deficit = atomic_readandclear_int(&vm_pageout_deficit); + page_shortage = vm_paging_target() + deficit; + inactive_page_surplus = vm_pageout_get_inactive_page_surplus(); + } else + page_shortage = deficit = inactive_page_surplus = 0; + starting_page_shortage = page_shortage; + + /* * If we need to reclaim memory ask kernel caches to return * some. We rate limit to avoid thrashing. */ - if (vmd == &vm_dom[0] && pass > 0 && + if (vmd == &vm_dom[0] && pass > 0 && page_shortage > 0 && (time_uptime - lowmem_uptime) >= lowmem_period) { /* * Decrease registered cache sizes. @@ -894,26 +981,6 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) } /* - * The addl_page_shortage is the number of temporarily - * stuck pages in the inactive queue. In other words, the - * number of pages from the inactive count that should be - * discounted in setting the target for the active queue scan. - */ - addl_page_shortage = 0; - - /* - * Calculate the number of pages that we want to free. This number - * can be negative if many pages are freed between the wakeup call to - * the page daemon and this calculation. - */ - if (pass > 0) { - deficit = atomic_readandclear_int(&vm_pageout_deficit); - page_shortage = vm_paging_target() + deficit; - } else - page_shortage = deficit = 0; - starting_page_shortage = page_shortage; - - /* * maxlaunder limits the number of dirty pages we flush per scan. * For most systems a smaller value (16 or 32) is more robust under * extreme memory and disk pressure because any unnecessary writes @@ -928,10 +995,18 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) if (pass > 1) maxlaunder = 10000; + /* + * Prevent laundering if there's no page shortage and we are + * merely trying to free inactive pages. Otherwise we may end + * up swapping before it's really necessary. + */ + if (page_shortage <= 0) + maxlaunder = 0; + vnodes_skipped = 0; - SDT_PROBE4(vm, , , before__inactive__scan, vmd, pass, page_shortage, - deficit); + SDT_PROBE5(vm, , , before__inactive__scan, vmd, pass, page_shortage, + deficit, inactive_page_surplus); /* * Start scanning the inactive queue for pages that we can free. The @@ -944,7 +1019,8 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) vm_pagequeue_lock(pq); queue_locked = TRUE; for (m = TAILQ_FIRST(&pq->pq_pl); - m != NULL && maxscan-- > 0 && page_shortage > 0; + m != NULL && maxscan-- > 0 && + (page_shortage > 0 || inactive_page_surplus > 0); m = next) { vm_pagequeue_assert_locked(pq); KASSERT(queue_locked, ("unlocked inactive queue")); @@ -1080,6 +1156,7 @@ free_page: vm_page_free(m); PCPU_INC(cnt.v_dfree); --page_shortage; + --inactive_page_surplus; } else if ((object->flags & OBJ_DEAD) != 0) { /* * Leave dirty pages from dead objects at the front of @@ -1107,13 +1184,19 @@ requeue_page: vm_pagequeue_lock(pq); queue_locked = TRUE; vm_page_requeue_locked(m); - } else if (maxlaunder > 0) { + } else if (maxlaunder > 0 && page_shortage > 0) { /* - * We always want to try to flush some dirty pages if - * we encounter them, to keep the system stable. + * As long as there is a page shortage, we try to + * flush some dirty pages if we encounter them, to + * keep the system stable. * Normally this number is small, but under extreme * pressure where there are insufficient clean pages * on the inactive queue, we may have to go all out. + * + * XXX: We probably should not bother laundering + * until we know that there might be a chance + * that we will not be able to free the required + * amount of pages to take care of the page shortage. */ if (object->type != OBJT_SWAP && @@ -1158,15 +1241,27 @@ relock_queue: } vm_pagequeue_unlock(pq); - SDT_PROBE5(vm, , , after__inactive__scan, vmd, pass, page_shortage, - addl_page_shortage, vnodes_skipped); + /* + * If the page shortage has been taken care of, or if we were + * just trying to free surplus inactive pages, the locked pages + * are more or less meaningless. Reset the counter to prevent + * pointless swapping. + */ + if (page_shortage <= 0) + addl_page_shortage = 0; + + SDT_PROBE6(vm, , , after__inactive__scan, vmd, pass, page_shortage, + addl_page_shortage, vnodes_skipped, inactive_page_surplus); #if !defined(NO_SWAPPING) /* - * Wakeup the swapout daemon if we didn't free the targeted number of - * pages. + * Wakeup the swapout daemon if we didn't free the targeted number + * of pages and we are either desperate or there are no inactive + * pages to free left (in which case we will be desperate soon + * enough). */ - if (vm_swap_enabled && page_shortage > 0) + if (vm_swap_enabled && page_shortage > 0 && + (pass > 1 || !vm_pageout_get_inactive_page_surplus())) vm_req_vmdaemon(VM_SWAP_NORMAL); #endif @@ -1587,7 +1682,9 @@ vm_pageout_worker(void *arg) /* * Might the page daemon receive a wakeup call? */ - if (vm_pageout_wanted) { + /* XXX: After r300865 this may no longer work. Investigate! */ + if (vm_pageout_wanted || + vm_pageout_get_inactive_page_surplus() > 0) { /* * No. Either vm_pageout_wanted was set by another * thread during the previous scan, which must have @@ -1656,6 +1753,29 @@ vm_pageout_init(void) vm_cnt.v_inactive_target = vm_cnt.v_free_count / 3; /* + * Apply the inactive memory limit before ZFS's + * dirty data limit kicks in. + */ + inactive_page_limit_offset_f = vm_cnt.v_page_count / 9 + - vm_cnt.v_free_target; + + /* + * A man carrying a large stone through the savanne was asked why + * he would do such a strange thing. His explanation: "If a lion + * comes, I'll through away the stone which will allow me to run + * faster." + * + * For similar reasons the inactive page limit defaults to + * allowing twice the number of inactive pages the vm targets + * itself. + * + * If sudden memory pressure comes, the inactive page reserve can + * be thrown away to make it less likely that the system has to + * start paging. + */ + inactive_page_limit_offset_i = vm_cnt.v_inactive_target; + + /* * Set the default wakeup threshold to be 10% above the minimum * page limit. This keeps the steady state out of shortfall. */ -- 2.11.0 From d6fd9b3e4e16a6e1d158e085217f8c6ba50738f6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 27 May 2016 11:56:18 +0200 Subject: [PATCH 240/257] ZFS: Default to reducing the dirty data buffer to 50% of the maximum It's currently unclear whether or not dynamically scaling the buffer is a good idea at all. Obtained from: ElectroBSD --- .../contrib/opensolaris/uts/common/fs/zfs/arc.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c index 61b1bcdffef2..477a528e2411 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c @@ -6277,17 +6277,23 @@ arc_init(void) zfs_dirty_data_max_max); } /* - * If memory is tight, reduce the dynamic write buffer - * as low as 0.5% of the physical memory. This allows - * a system with 1 GB of physical memory to remain stable - * while building kernels in a loop for days. - * Using 1% (10 MB) proved to be too much. + * Calculate minimum amount of dirty data allowed per pool. * - * Systems with 2 GB of physical memory don't seem to - * reach the lower limit while building kernels. + * Initially the default was 0.5% of the system's memory + * and was supposed to prevent crashes when reproducing + * ElectroBSD on systems with 1 GB or less. + * + * While it made the crashes less common, it did not actually + * prevent them. + * + * The problem was eventually tracked down to a geli bug + * and fixed (#209759), therefore it's unclear if dynamically + * reducing the amount of dirty data allowed is still useful. + * + * For now it's set to 50% of the maximum. */ if (zfs_dirty_data_max_min == 0) { - zfs_dirty_data_max_min = ptob(physmem) / 200; + zfs_dirty_data_max_min = zfs_dirty_data_max / 2; } #ifdef _KERNEL -- 2.11.0 From e4c662343ef8da1a8fd47f4f1c8c4e355cfaeacd Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 23 Oct 2015 23:36:28 +0200 Subject: [PATCH 241/257] zfs: Do not advertise edonr which isn't supported yet illumos 4185 ("add new cryptographic checksums to ZFS: SHA-512, Skein, Edon-R") was intentionally merged only partially in r289422, without adding support for skein, sha512 and edonr on FreeBSD. Support for skein and sha512 was added later on, but edonr remains missing. Without this commit, zfs(8) correctly rejects edonr, the error message claims that it is supported, though: fk@r500 ~ $zfs set checksum=edonr tank cannot set property for 'tank': 'checksum' must be one of 'on | off | fletcher2 | fletcher4 | sha256 | sha512 | skein | edonr' Obtained from: ElectroBSD PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=204055 PR submission date: 2015-10-27 (before sha512 and skein were added) PR update date: Not yet updated as upstream does not seem to care anyway. --- sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c b/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c index 079d86043f4d..2a4b18abb2a0 100644 --- a/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c +++ b/sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c @@ -242,11 +242,11 @@ zfs_prop_init(void) ZIO_CHECKSUM_DEFAULT, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, "on | off | fletcher2 | fletcher4 | sha256 | sha512 | " - "skein | edonr", "CHECKSUM", checksum_table); + "skein", "CHECKSUM", checksum_table); zprop_register_index(ZFS_PROP_DEDUP, "dedup", ZIO_CHECKSUM_OFF, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, "on | off | verify | sha256[,verify], sha512[,verify], " - "skein[,verify], edonr,verify", "DEDUP", dedup_table); + "skein[,verify]", "DEDUP", dedup_table); zprop_register_index(ZFS_PROP_COMPRESSION, "compression", ZIO_COMPRESS_DEFAULT, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, -- 2.11.0 From f9c53947e57d2a1a40b069d902de8cbbd2fd98d8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 29 May 2015 10:46:06 +0200 Subject: [PATCH 242/257] Import cloudiatr 2016-11-02-89a8898 Obtained from: ElectroBSD --- usr.sbin/Makefile | 1 + usr.sbin/cloudiatr/Makefile | 3 + usr.sbin/cloudiatr/cloudiatr | 1275 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1279 insertions(+) create mode 100644 usr.sbin/cloudiatr/Makefile create mode 100755 usr.sbin/cloudiatr/cloudiatr diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile index 3d146d6e7704..1b739878676d 100644 --- a/usr.sbin/Makefile +++ b/usr.sbin/Makefile @@ -8,6 +8,7 @@ SUBDIR= adduser \ binmiscctl \ bsdconfig \ camdd \ + cloudiatr \ cdcontrol \ chkgrp \ chown \ diff --git a/usr.sbin/cloudiatr/Makefile b/usr.sbin/cloudiatr/Makefile new file mode 100644 index 000000000000..34ada8582ecd --- /dev/null +++ b/usr.sbin/cloudiatr/Makefile @@ -0,0 +1,3 @@ +SCRIPTS= cloudiatr + +.include diff --git a/usr.sbin/cloudiatr/cloudiatr b/usr.sbin/cloudiatr/cloudiatr new file mode 100755 index 000000000000..461815d7e9c3 --- /dev/null +++ b/usr.sbin/cloudiatr/cloudiatr @@ -0,0 +1,1275 @@ +#!/bin/sh + +########################################################################### +# cloudiatr +# +# Buzzword-compliant remote OS eviction tool. For details see: +# https://www.fabiankeil.de/gehacktes/cloudiatr/ +# +# Copyright (c) 2014-2016 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ALL YOUR +# DATA IS BELONG TO THE SOFTWARE AND MAY BE EATEN BY IT. IF THAT IS NOT +# ACCEPTABLE, YOU SHOULD PROBABLY MAKE BACKUPS BEFORE USING THE SOFTWARE. +########################################################################### + +# It's important that this function is called before any other +# function except cloudiatr_main(), otherwise fatal errors may +# not be caught. +cloudiatr_init() { + local mode="${1}" + + set -e + cloudiatr_init_globals + + cloudiatr_load_config_file "${CLOUDIATR_CONFIG_FILE}" + + cloudiatr_check_config +} + +cloudiatr_fyi() { + local message="${*}" + + echo "cloudiatr: $message" +} + +cloudiatr_wtf() { + local complaints="${*}" + if [ -z "${complaints}" ]; then + complaints="cloudiatr_wtf(): No complaints?" + fi + cloudiatr_fyi "${complaints}" 1>&2 + return 1 +} + +cloudiatr_check_config() { + local v_flag \ + mandatory_variable optional_variable value fail + + v_flag="${1}" + fail=0 + + for mandatory_variable in ${CLOUDIA_MANDATORY_VARIABLES}; do + value="$(eval 'echo $'"${mandatory_variable}")" + if [ -z "${value}" ]; then + cloudiatr_wtf "Fatal error: ${mandatory_variable} is unset" + fail=1 + elif [ "${v_flag}" = "-v" ]; then + echo "${mandatory_variable}='${value}'" + fi + done + if [ "${v_flag}" = "-v" ]; then + for optional_variable in ${CLOUDIA_OPTIONAL_VARIABLES}; do + value="$(eval 'echo $'"${optional_variable}")" + echo "${optional_variable}='${value}'" + done + fi + return $fail +} + +cloudiatr_show_config() { + cloudiatr_check_config -v +} + +cloudiatr_load_config_file() { + local config_file="${1}" + + if [ -f "${config_file}" ]; then + . "${config_file}" + return 0 + fi + cloudiatr_wtf "Config file ${config_file} does not exist. You can use '$0 -f path/to/file ...' to specify a different one" + return 1 +} + +cloudiatr_init_globals() { + + CLOUDIATR_VERSION="2016-11-02-89a8898" + + CLOUDIATR_NEW_SYSTEM_DIR=/cloudiatr + # Only needs to be enough for a stripped-down bootfs + CLOUDIATR_BPOOL_PARTITION_SIZE=200M + # Has to be enough for the rest of the OS including the "permanent" + # /boot that is only used to (re)populate the bootfs on the bpool. + CLOUDIATR_RPOOL_PARTITION_SIZE=4G + CLOUDIATR_SWAP_PARTITION_SIZE=4G + + # Set to true to use the existing partition layout. + # + # Only expected to work if the layout was created by a previous + # cloudiatr run. By default the partitions 2, 3 and 4 will be + # overwritten and partition 1 is expected to contain working bootcode. + CLOUDIATR_REUSE_GPART_SETUP=false + + # Usually changing the partition numbers usually is not necessary + # and changing them after the installation is likely to result in + # data loss. + CLOUDIATR_BOOTCODE_PARTITION=1 + CLOUDIATR_BPOOL_PARTITION=2 + CLOUDIATR_RPOOL_PARTITION=3 + CLOUDIATR_SWAP_PARTITION=4 + CLOUDIATR_DPOOL_PARTITION=5 + + CLOUDIATR_BPOOL_NAME="bpool" + CLOUDIATR_RPOOL_NAME="rpool" + + # Note that DEFAULT is a fallback documented in rc.conf(5). + # Not changing this variable to the actual network interface + # is likely to cause problems if there are more than one nics + # and you aren't using DHCP for all of them. + CLOUDIATR_NIC="DEFAULT" + + # Will be created + CLOUDIATR_RPOOL_KEY_NAME="${CLOUDIATR_RPOOL_NAME}.key" + CLOUDIATR_RPOOL_KEY="${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_KEY_NAME}" + + # If CLOUDIATR_DIST_IMAGE is set, an image containing the dist tarballs + # has to be put in place by the user before the eviction. If it's unset, + # CLOUDIATR_DIST_DIR has to be populated before cloudiatr is executed. + CLOUDIATR_DIST_IMAGE="" + CLOUDIATR_DIST_IMAGE_SHA256="" + CLOUDIATR_DIST_DIR="/usr/electrobsd-dist/" + + # Whether or not the distribution tarballs should be copied to the + # newly installed system (for example to reuse them when setting + # up jails). + CLOUDIATR_SAVE_DIST_DIR="false" + + # Default to using all the detected ada(4) devices + CLOUDIATR_DISKS="$(cloudiatr_autodetect_disks)" + + # Changing these should only be necessary if there's more + # than one disk and you don't want to a mirror. + CLOUDIATR_BPOOL_LAYOUT="default" + CLOUDIATR_RPOOL_LAYOUT="default" + + CLOUDIATR_GELI_KEY_LENGTH=256 + CLOUDIATR_GELI_EALGO=AES-XTS + + CLOUDIATR_SSHD_HOST_KEY_ALGORITHMS="rsa ecdsa ed25519" + + # Set to 'true' to ingore some safety-checks and increase the potential damage. + # Includes "geli kill -a" which is not limited to the disks specified above. + # Do not enable this unless the system that is being evicted doesn't contain + # any data you care about. + CLOUDIATR_MURDER_DEATH_KILL_REQUESTED=false + + # Set to 'true' to skip the image checksum check. + # "It may be insecure, but look how fast it is!" + CLOUDIATR_CHECKSUM_SMECKSUM=false + + # The config file is sourced and may overwrite any of the values above + # and most functions in this file (zogftw-style). + CLOUDIATR_CONFIG_FILE="${CLOUDIATR_CONFIG_FILE=/etc/cloudiatr.conf}" + + # A user that should be created and allowed to "su" on the new system. + CLOUDIATR_NEW_USER="cloudiatr" + + # This password is used for both root and CLOUDIATR_NEW_USER. + # + # Note that the created system will not accept root logins through + # ssh (FreeBSD default). CLOUDIATR_NEW_USER may use ssh, but has + # to use public key authentication. + CLOUDIATR_INITIAL_PASSWORD="${CLOUDIATR_NEW_USER}" + + # Local timezone. For details see tzsetup(8). + CLOUDIATR_TIME_ZONE="Europe/Berlin" + + # When set to true, cloudiatr will execute ntpdate at installtime. + # The server(s) being used depend on the install distfiles. + CLOUDIATR_USE_NTPDATE="false" + + # Any alignment should work, 1M is often recommended to prevent + # write-amplification which can result in performance degradation. + # It can also be advantageous for trimming SSDs. + CLOUDIATR_GPART_ALIGNMENT="1M" + + # Optional keyboard map for the virtual console. + # For details see kbdmap. + CLOUDIATR_KBDMAP="de.kbd" + + # Additional distributions to extract. Example: src, lib32 + CLOUDIATR_EXTRA_DISTRIBUTIONS="" + + # Value for rc.conf's rether_enable entry which controls + # whether or not MAC addresses are randomized (on ElectroBSD). + CLOUDIATR_RETHER_ENABLE="NO" + + # Set to true to not bother the user about with questions. + : "${CLOUDIATR_DONT_ASK_JUST_KISS=false}" + + # Set to true (default) to use the added swap partitions right + # after creating them. This allows installations on systems that + # have insufficient memory (512 MB, for example) and no previously + # configured swap devices. + # + # While this option is not expected to cause problems, if you + # are absolutely sure that enough memory is available you can + # disable the behaviour by setting the variable to "false". + CLOUDIATR_USE_SWAP_WHILE_INSTALLING="true" + + # Apply a workaround that is required to boot on + # Lenovo laptops like the T520 + CLOUDIATR_APPLY_LENOVO_WORKAROUND="false" + + # If these variables aren't set to some value, cloudiatr will abort. + # Sane values are a good idea but not mandatory. + CLOUDIA_MANDATORY_VARIABLES="\ + CLOUDIATR_APPLY_LENOVO_WORKAROUND \ + CLOUDIATR_BPOOL_LAYOUT \ + CLOUDIATR_BPOOL_NAME \ + CLOUDIATR_BPOOL_PARTITION_SIZE \ + CLOUDIATR_CHECKSUM_SMECKSUM \ + CLOUDIATR_CONFIG_FILE \ + CLOUDIATR_DISKS \ + CLOUDIATR_DIST_DIR \ + CLOUDIATR_DONT_ASK_JUST_KISS \ + CLOUDIATR_GELI_KEY_LENGTH \ + CLOUDIATR_GELI_EALGO \ + CLOUDIATR_GPART_ALIGNMENT \ + CLOUDIATR_HOSTNAME \ + CLOUDIATR_INITIAL_PASSWORD \ + CLOUDIATR_MURDER_DEATH_KILL_REQUESTED \ + CLOUDIATR_NEW_SYSTEM_DIR \ + CLOUDIATR_NEW_USER \ + CLOUDIATR_NIC \ + CLOUDIATR_USE_SWAP_WHILE_INSTALLING \ + CLOUDIATR_RETHER_ENABLE \ + CLOUDIATR_REUSE_GPART_SETUP \ + CLOUDIATR_RPOOL_KEY \ + CLOUDIATR_RPOOL_KEY_NAME \ + CLOUDIATR_RPOOL_LAYOUT \ + CLOUDIATR_RPOOL_NAME \ + CLOUDIATR_RPOOL_PARTITION_SIZE \ + CLOUDIATR_SAVE_DIST_DIR \ + CLOUDIATR_SWAP_PARTITION_SIZE \ + CLOUDIATR_TIME_ZONE \ + CLOUDIATR_USE_NTPDATE \ + CLOUDIATR_VERSION \ + " + + # These variables are allowed to be unset + CLOUDIA_OPTIONAL_VARIABLES="\ + CLOUDIATR_DEFAULTROUTER \ + CLOUDIATR_DIST_IMAGE \ + CLOUDIATR_DIST_IMAGE_SHA256 \ + CLOUDIATR_EXTRA_DISTRIBUTIONS \ + CLOUDIATR_IP_ADDRESS \ + CLOUDIATR_KBDMAP \ + CLOUDIATR_NETMASK \ + " +} + +# Apply the workaround described at: +# https://lists.freebsd.org/pipermail/freebsd-i386/2013-March/010437.html +cloudiatr_apply_lenovo_workaround() { + local disk="${1}" \ + slice_table_original slice_table_new \ + partion_spec cylinders sectors_per_track cylinders_new heads + + if [ -z "${disk}" ]; then + cloudiatr_wtf "cloudiatr_apply_lenovo_workaround: No disk provided" + return 1 + fi + + slice_table_original=$(mktemp -t cloudiatr_slice_table_original) || return 1 + slice_table_new=$(mktemp -t cloudiatr_slice_table_new) || return 1 + + fdisk -p "${disk}" > "${slice_table_original}" || return 1 + + cylinders="$(grep '^g c' "${slice_table_original}" | cut -w -f 2 | cut -d c -f 2)" || return 1 + heads="$(grep '^g c' "${slice_table_original}" | cut -w -f 3 | cut -d h -f 2)" || return 1 + sectors_per_track="$(grep '^g c' "${slice_table_original}" | cut -w -f 4 | cut -d s -f 2)" || return 1 + + cylinders_new="$(expr "${cylinders}" \* "${sectors_per_track}")" || return 1 + + partition_spec="$(tail -n 1 ${slice_table_original})" || return 1 + + echo "# ${disk}" > "${slice_table_new}" || return 1 + echo "g c${cylinders_new} h${heads} s1" >> "${slice_table_new}" || return 1 + echo "${partition_spec}" | sed -e 's@^p 1 0xee@p 1 0x00@' >> "${slice_table_new}" || return 1 + echo "a 1" >> "${slice_table_new}" || return 1 + echo "${partition_spec}" | sed -e 's@^p 1@p 2@' >> "${slice_table_new}" || return 1 + + cloudiatr_fyi "Applying Lenovo workaround:" + diff -u "${slice_table_original}" "${slice_table_new}" || true + fdisk -f "${slice_table_new}" "${disk}" || return 1 + rm "${slice_table_original}" "${slice_table_new}" || return 1 +} + +cloudiatr_gpart_disk() { + local disk="${1}" \ + disk_name + + disk_name="${disk##*/}" + + gpart create -s gpt "${disk}" || return 1 + + gpart add -s 512 -t freebsd-boot \ + -i "${CLOUDIATR_BOOTCODE_PARTITION}" "${disk}" || return 1 + gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot \ + -i "${CLOUDIATR_BOOTCODE_PARTITION}" "${disk}" || return 1 + + gpart add -s "${CLOUDIATR_BPOOL_PARTITION_SIZE}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -l "${CLOUDIATR_BPOOL_NAME}-${disk_name}" -t freebsd-zfs \ + -i "${CLOUDIATR_BPOOL_PARTITION}" "${disk}" || return 1 + gpart add -s "${CLOUDIATR_RPOOL_PARTITION_SIZE}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -l "${CLOUDIATR_RPOOL_NAME}-${disk_name}" -t freebsd-zfs \ + -i "${CLOUDIATR_RPOOL_PARTITION}" "${disk}" || return 1 + gpart add -s "${CLOUDIATR_SWAP_PARTITION_SIZE}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -l "swap-${disk_name}" -t freebsd-swap \ + -i "${CLOUDIATR_SWAP_PARTITION}" "${disk}" || return 1 + + # Reserve what's left for the data pool + gpart add -l "dpool-${disk_name}" -a "${CLOUDIATR_GPART_ALIGNMENT}" \ + -t freebsd-zfs -i "${CLOUDIATR_DPOOL_PARTITION}" "${disk}" || return 1 + + if "${CLOUDIATR_APPLY_LENOVO_WORKAROUND}"; then + cloudiatr_apply_lenovo_workaround "${disk}" || return 1 + fi +} + +cloudiatr_gpart_setup() { + local disks d + + disks="${*}" + + cloudiatr_fyi "Cleaning partition tables (if there are any) ..." + for d in $disks; do + gpart destroy -F "${d}" 2>/dev/null || true + done + + cloudiatr_fyi "Partitioning disks ..." + for d in $disks; do + cloudiatr_gpart_disk "${d}" || return 1 + done +} + +# Use the swap partitions on the given disks while cloudiatr is running. +# This allows to install on a system with 512MB RAM or less and no swap space. +cloudiatr_enable_swap() { + local disks d + + disks="${*}" + + cloudiatr_fyi "Using created swap space while installing ..." + for d in $disks; do + geli onetime -d "${d}p${CLOUDIATR_SWAP_PARTITION}" || return 1 + swapon "${d}p${CLOUDIATR_SWAP_PARTITION}.eli" || return 1 + done +} + +cloudiatr_disable_swap() { + local disks d + + disks="${*}" + + cloudiatr_fyi "Trying to disable previously added swap space ..." + for d in $disks; do + swapoff "${d}p${CLOUDIATR_SWAP_PARTITION}.eli" || return 1 + done +} + +cloudiatr_get_geoms() { + local postfix="${1}" \ + d geoms + + for d in ${CLOUDIATR_DISKS}; do + geom_partition="${d}${postfix}" + geoms="${geoms} ${geom_partition}" + done + echo "${geoms## }" +} + +cloudiatr_get_gpart_labels() { + local postfix="${1}" + + for d in ${CLOUDIATR_DISKS}; do + d="${d##/dev/}" + gpart show -l -p "${d}" 2>/dev/null | awk '$3 == "'"${d}${postfix}"'" {printf "%s ", $4}' + done + echo +} + +# Depends on geli being already setup +cloudiatr_create_rpool() { + local \ + rpool_elis pool_layout + + rpool_elis="$(cloudiatr_get_geoms "p${CLOUDIATR_RPOOL_PARTITION}.eli")" + pool_layout="${CLOUDIATR_RPOOL_LAYOUT}" + + if [ "${pool_layout}" = "default" ]; then + pool_layout="$(cloudiatr_get_default_pool_layout)" + fi + + cloudiatr_fyi "Creating root pool '${CLOUDIATR_RPOOL_NAME}' on ${rpool_elis}. Pool layout: ${pool_layout}" + + zpool create -o version=28 -o failmode=continue \ + -O compression=lzjb -O checksum=sha256 \ + "${CLOUDIATR_RPOOL_NAME}" ${pool_layout##single-disk} $rpool_elis + + zfs set mountpoint="${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}" "${CLOUDIATR_RPOOL_NAME}" + + zfs create "${CLOUDIATR_RPOOL_NAME}/boot" + # We currently use no dedicated dataset for /etc as the kernel expects parts of it + # to be available once the rootfs has been mounted. Having two /etc's can be a bit + # of a hassle on updates and thus doesn't seem like a good default. + #zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/etc" + zfs create "${CLOUDIATR_RPOOL_NAME}/home" + zfs create "${CLOUDIATR_RPOOL_NAME}/home/${CLOUDIATR_NEW_USER}" + zfs create -o exec=on -o setuid=off "${CLOUDIATR_RPOOL_NAME}/tmp" + zfs create "${CLOUDIATR_RPOOL_NAME}/usr" + zfs create -o compression=gzip -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/src" + zfs create "${CLOUDIATR_RPOOL_NAME}/var" + zfs create "${CLOUDIATR_RPOOL_NAME}/usr/local" + zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/local/etc" + zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/local/src" + zfs create -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/ports" + zfs create -o compression=off -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/ports/distfiles" + zfs create -o compression=off -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/usr/ports/packages" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/crash" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/db" + zfs create -o exec=on -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/db/pkg" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/empty" + zfs create -o compression=gzip -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/log" + zfs create -o compression=gzip -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/mail" + zfs create -o exec=off -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/run" + zfs create -o exec=on -o setuid=off "${CLOUDIATR_RPOOL_NAME}/var/tmp" + + chmod 0750 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/crash" + chgrp mail "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/mail" + chmod 0775 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/mail" + chmod 0555 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/empty" + chflags schg,nouarch "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/empty" + chmod 1777 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/var/tmp" + chmod 1777 "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/tmp" + + zfs set readonly=on "${CLOUDIATR_RPOOL_NAME}/var/empty" +} + +cloudiatr_kernel_module_is_loaded() { + local module="${1}" + # Can't use 'kldstat -m foo' as it requires a module file on disk. WTF? + kldstat | grep -q "${module}" +} + +cloudiatr_mount_dist_image() { + # intentionally leaks non-local variable md + + if [ ! -f "${CLOUDIATR_DIST_IMAGE}" ]; then + cloudiatr_wtf "File ${CLOUDIATR_DIST_IMAGE} does not exist" + return 1 + fi + + if "${CLOUDIATR_CHECKSUM_SMECKSUM}"; then + cloudiatr_fyi "Checksum smecksum" + else + cloudiatr_fyi "Checking checksum for image file ${CLOUDIATR_DIST_IMAGE} ..." + sha256 -c "${CLOUDIATR_DIST_IMAGE_SHA256}" "${CLOUDIATR_DIST_IMAGE}" + fi + md=$(mdconfig -f "${CLOUDIATR_DIST_IMAGE}") + if [ -z "${md}" ]; then + return 1 + fi + + for potential_partition in "/dev/${md}a" "/dev/${md}p2"; do + if [ -c "${potential_partition}" ]; then + cloudiatr_fyi "Trying to mount ${potential_partition} ..." + mount -o ro "${potential_partition}" /mnt/ || return 1 + cloudiatr_fyi "Mounting ${potential_partition} worked..." + fi + done +} + +cloudiatr_extract_distribution() { + local chroot_dir \ + md extra_distribution + + chroot_dir="${1}" + + if [ -n "${CLOUDIATR_DIST_IMAGE}" ]; then + cloudiatr_mount_dist_image + else + cloudiatr_fyi "No CLOUDIATR_DIST_IMAGE specified." + cloudiatr_fyi "Using CLOUDIATR_DIST_DIR=${CLOUDIATR_DIST_DIR}!" + fi + + cloudiatr_fyi "Extracting base in ${chroot_dir} ..." + # Exclude /var/empty as it's read-only + (cd "${chroot_dir}" && tar xpf "${CLOUDIATR_DIST_DIR}/base.txz" --exclude ./var/empty/) + + cloudiatr_fyi "Creating directories that were missing in the base tarball" + chroot "${chroot_dir}" mtree -f /etc/mtree/BSD.root.dist -d -e -u + chroot "${chroot_dir}" mtree -f /etc/mtree/BSD.var.dist -d -e -u -p var + + for extra_distribution in ${CLOUDIATR_EXTRA_DISTRIBUTIONS}; do + cloudiatr_fyi "Extracting extra distribution '${extra_distribution}'" + (cd "${chroot_dir}" && tar xpf "${CLOUDIATR_DIST_DIR}/${extra_distribution}.txz") + done + + cloudiatr_fyi "Extracting kernel (without symbols) in ${chroot_dir} ..." + (cd "${chroot_dir}" && tar xpf "${CLOUDIATR_DIST_DIR}/kernel.txz" --exclude "*.symbols") + + if [ -n "${CLOUDIATR_DIST_IMAGE}" ]; then + umount /mnt + mdconfig -d -u ${md##md} + fi +} + +cloudiatr_setup_new_user() { + local ssh_dir="${chroot_dir}/home/${CLOUDIATR_NEW_USER}/.ssh" + + cloudiatr_fyi "Creating user '${CLOUDIATR_NEW_USER}'" + + echo "${CLOUDIATR_INITIAL_PASSWORD}" | chroot "${chroot_dir}" \ + pw useradd "${CLOUDIATR_NEW_USER}" -G wheel,operator -h 0 + + # Make sure the user can login through ssh, using the + # authorized_keys file from the installation media. + # + # An extra distribution file may already have created + # the .ssh directory, so don't fail if it already exits. + mkdir -p "${ssh_dir}" + cp -v "${HOME}/.ssh/authorized_keys" "${ssh_dir}" || true + chroot "${chroot_dir}" chown -R "${CLOUDIATR_NEW_USER}" "/home/${CLOUDIATR_NEW_USER}" + chroot "${chroot_dir}" chmod -R go-rwx "/home/${CLOUDIATR_NEW_USER}" +} + +cloudiatr_create_geli_key() { + local keyfile="${1}" + + ( + umask 077 + dd bs=64 count=1 if=/dev/random of="${keyfile}" 2>/dev/null + ) +} + +cloudiatr_setup_geli() { + local disks \ + d + + disks="${*}" + + cloudiatr_create_geli_key "${CLOUDIATR_RPOOL_KEY}" + + mkdir "${CLOUDIATR_NEW_SYSTEM_DIR}/geli-backups" + + for d in $disks; do + cloudiatr_fyi "Initialising geli on ${d}p${CLOUDIATR_RPOOL_PARTITION} ..." + geli init -b \ + -B "${CLOUDIATR_NEW_SYSTEM_DIR}/geli-backups/${d##/dev/}p${CLOUDIATR_RPOOL_PARTITION}.eli" \ + -P -K "${CLOUDIATR_RPOOL_KEY}" -l "${CLOUDIATR_GELI_KEY_LENGTH}" \ + -e "${CLOUDIATR_GELI_EALGO}" -s 4096 -V 7 "${d}p${CLOUDIATR_RPOOL_PARTITION}" > /dev/null + done + + cloudiatr_attach_geli_geoms "${CLOUDIATR_RPOOL_KEY}" "p${CLOUDIATR_RPOOL_PARTITION}" +} + +cloudiatr_attach_geli_geoms() { + local keyfile partition_id \ + disk + + keyfile="${1}" + partition_id="${2}" + + for disk in ${CLOUDIATR_DISKS}; do + cloudiatr_fyi "geli-attaching ${disk}${partition_id}" + geli attach -p -k "$keyfile" "${disk}${partition_id}" + done +} + +cloudiatr_detach_geli_geoms() { + local partition_id + + partition_id="${1}" + + for disk in ${CLOUDIATR_DISKS}; do + cloudiatr_fyi "geli-detaching ${disk}${partition_id}" + geli detach "${disk}${partition_id}.eli" + done +} + +cloudiatr_get_bpool_geoms() { + cloudiatr_get_geoms "p${CLOUDIATR_BPOOL_PARTITION}" +} + +cloudiatr_get_disk_names() { + local disk + for disk in ${CLOUDIATR_DISKS}; do + echo "${disk##*/}" + done +} + +cloudiatr_get_number_of_disks() { + local \ + disk number_of_disks + + number_of_disks=0 + for disk in ${CLOUDIATR_DISKS}; do + number_of_disks=$((number_of_disks+1)) + done + echo "${number_of_disks}" +} + +cloudiatr_get_default_pool_layout() { + if [ "$(cloudiatr_get_number_of_disks)" = 1 ]; then + echo "single-disk" + else + echo "mirror" + fi +} + +cloudiatr_autodetect_disks() { + local \ + disk + + for disk in $(sysctl -n kern.disks); do + # Only use ada(4) devices. We obviously can't use cd(4) + # devices and using da(4) devices would require us to + # skip the one we (probably) booted from. + if [ "${disk##ada}" != "${disk}" ]; then + echo "/dev/${disk}" + fi + done +} + +cloudiatr_create_bpool() { + local \ + bpool_geoms pool_layout + + bpool_geoms="$(cloudiatr_get_bpool_geoms)" + pool_layout="${CLOUDIATR_BPOOL_LAYOUT}" + + if [ "${pool_layout}" = "default" ]; then + pool_layout="$(cloudiatr_get_default_pool_layout)" + fi + + cloudiatr_fyi "Creating boot pool '${CLOUDIATR_BPOOL_NAME}' on ${bpool_geoms}. Pool layout: ${pool_layout}" + + zpool create -f -o version=28 -O compression=lzjb \ + "${CLOUDIATR_BPOOL_NAME}" ${pool_layout##single-disk} \ + $bpool_geoms + + # This currently can't be set at create-time + zpool set "bootfs=${CLOUDIATR_BPOOL_NAME}" "${CLOUDIATR_BPOOL_NAME}" + + # Would be nice, but for the bootfs to work, + # its ./boot directory can't be a zfs fs. + # + # XXX: can we work around this by setting bootfs + # on bpool/boot and use a symlink from bootf/boot/boot + # to bootf/boot? + # zfs create "${CLOUDIATR_BPOOL_NAME}/boot" +} + +cloudiatr_setup_tmpfs() { + mkdir -p "${CLOUDIATR_NEW_SYSTEM_DIR}" + mount -t tmpfs tmpfs "${CLOUDIATR_NEW_SYSTEM_DIR}" +} + +cloudiatr_generate_rc_conf() { + local \ + netmask + + cat < "${config_file}" +} + +cloudiatr_create_config_files() { + local chroot_dir="${1}" + + cloudiatr_generate_file loader_conf "${chroot_dir}/boot/loader.conf" + cloudiatr_generate_file sysctl_conf "${chroot_dir}/etc/sysctl.conf" + cloudiatr_generate_file rc_conf "${chroot_dir}/etc/rc.conf" + cloudiatr_generate_file fstab "${chroot_dir}/etc/fstab" + cloudiatr_generate_file resolv_conf "${chroot_dir}/etc/resolv.conf" || true +} + +cloudiatr_get_required_kernel_content() { + kldstat | awk '/k/ {print $5}' +} + +# XXX: May creates output with duplicated slashes. Ugly but harmless. +# XXX: Why do we ignore errors here? +cloudiatr_populate_bpool() { + local boot_dir \ + boot_file new_kernel_dir sub_dir bpool_mountpoint new_file + + boot_dir="${1}" + if [ "${boot_dir}" = "/" ]; then + # Prevent duplicated leading slash in log messages + boot_dir="" + fi + bpool_mountpoint="${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_BPOOL_NAME}" + + cloudiatr_fyi "Populating boot pool '${CLOUDIATR_BPOOL_NAME}' ..." + zfs set mountpoint="${bpool_mountpoint}" "${CLOUDIATR_BPOOL_NAME}" || return 1 + + # Only copy what we really need before we can import the encrypted rpool + for sub_dir in dtb firmware kernel kernel.old zfs modules defaults; do + mkdir -p "${bpool_mountpoint}/boot/${sub_dir}" + done + + for boot_file in $(cloudiatr_get_required_kernel_content); do + # XXX: Create missing directories here + cp -v "${boot_dir}/boot/kernel/${boot_file}" "${bpool_mountpoint}/boot/kernel/" || true + if [ -f "${boot_dir}/boot/kernel.old/${boot_file}" ]; then + cp -v "${boot_dir}/boot/kernel.old/${boot_file}" "${bpool_mountpoint}/boot/kernel.old/" || true + fi + done + + for boot_file in $(find "${boot_dir}/boot/" \ + -not -path "${boot_dir}/boot/kernel*" -a \ + -not -path "${boot_dir}/boot/boot*" -a \ + -not -path "${boot_dir}/boot/cdboot" -a \ + -not -path "${boot_dir}/boot/*mbr" -a \ + -type f); do + new_file="${bpool_mountpoint}/boot/${boot_file##*/boot/}" + cp -v "${boot_file}" "${new_file}" || true + done + + zfs set "mountpoint=/${CLOUDIATR_BPOOL_NAME}" "${CLOUDIATR_BPOOL_NAME}" +} + +cloudiatr_clean_up() { + cloudiatr_fyi "Exporting boot pool '${CLOUDIATR_BPOOL_NAME}' ..." + zpool export "${CLOUDIATR_BPOOL_NAME}" + cloudiatr_fyi "Exporting root pool '${CLOUDIATR_RPOOL_NAME}' ..." + zpool export "${CLOUDIATR_RPOOL_NAME}" + cloudiatr_detach_geli_geoms "p${CLOUDIATR_RPOOL_PARTITION}" + umount "${CLOUDIATR_NEW_SYSTEM_DIR}" + rmdir "${CLOUDIATR_NEW_SYSTEM_DIR}" + if "${CLOUDIATR_USE_SWAP_WHILE_INSTALLING}"; then + if ! cloudiatr_disable_swap ${CLOUDIATR_DISKS}; then + cloudiatr_fyi "Failed to remove all the added swap space." + cloudiatr_fyi "If the system is low on memory the problem can be safely ignored." + fi + fi +} + +cloudiatr_generate_ssh_hostkeys() { + local chroot_dir \ + real_hostname + + chroot_dir="${1}" + real_hostname="$(hostname)" + + hostname "${CLOUDIATR_HOSTNAME}" + cloudiatr_fyi "Generating ssh host keys for ${CLOUDIATR_HOSTNAME} ..." + for key_alg in ${CLOUDIATR_SSHD_HOST_KEY_ALGORITHMS}; do + key_file="${chroot_dir}/etc/ssh/ssh_host_${key_alg}_key" + if ! ssh-keygen -q -t "${key_alg}" -f "${key_file}" -N ""; then + if [ "${key_alg}" = "ed25519" ]; then + # ed25519 isn't supported on FreeBSD 10.0 and earlier, + # thus we allow this to fail + continue + fi + return 1 + fi + ssh-keygen -l -v -f "${key_file}.pub" + done + hostname "${real_hostname}" +} + +cloudiatr_collect_evidence() { + local \ + evidence_dataset evidence disk_name + + evidence_dataset="${CLOUDIATR_RPOOL_NAME}/cloudiatr-evidence" + + cloudiatr_fyi "Collecting 'evidence' in /${evidence_dataset} ..." + zfs create "${evidence_dataset}" + for evidence in "${CLOUDIATR_CONFIG_FILE}" "${0}" \ + "${CLOUDIATR_NEW_SYSTEM_DIR}/geli-backups/"* \ + "${CLOUDIATR_RPOOL_KEY}"; do + cp -p "${evidence}" "${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}" + done + + for disk_name in $(cloudiatr_get_disk_names); do + evidence="${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}/${disk_name}.gpart" + gpart backup "${disk_name}" > "${evidence}" + done + + if "${CLOUDIATR_SAVE_DIST_DIR}"; then + cp -r "${CLOUDIATR_DIST_DIR%%/}" "${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}/" + fi + + find "${CLOUDIATR_NEW_SYSTEM_DIR}/${evidence_dataset}" -type f | sort +} + +cloudiatr_generate_sshd_config_extension() { + cat <> ${chroot_dir}/etc/ssh/sshd_config + + if "${CLOUDIATR_USE_NTPDATE}"; then + chroot ${chroot_dir} service ntpdate onestart || true + fi + + umount ${chroot_dir}/dev/ + + cloudiatr_setup_new_user + + # Copying the zpool.cache is no longer necessary on ElectroBSD + # and recent FreeBSD versions, but doesn't hurt. + cp /boot/zfs/zpool.cache "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}/boot/zfs/zpool.cache" + + cloudiatr_populate_bpool "${CLOUDIATR_NEW_SYSTEM_DIR}/${CLOUDIATR_RPOOL_NAME}" + + cloudiatr_collect_evidence + + cloudiatr_fyi "Setting final mountpoints on root pool '${CLOUDIATR_RPOOL_NAME}' ..." + zfs umount "${CLOUDIATR_RPOOL_NAME}" + zfs set mountpoint=legacy "${CLOUDIATR_RPOOL_NAME}" + for fs in boot home tmp usr var; do + zfs set "mountpoint=/${fs}" "${CLOUDIATR_RPOOL_NAME}/${fs}" + done + + cloudiatr_clean_up +} + +# The murder-death-kill feature was added for testing. After the introduction of +# the boring "clean-up" subcommand (which doesn't involve killing) it could be +# considered obsolete, but keeping it around makes cloudiatr more awesome. +cloudiatr_murder_death_kill() { + if "${CLOUDIATR_MURDER_DEATH_KILL_REQUESTED}"; then + cloudiatr_fyi "You really asked for it. Murder death kill in progress ..." + zpool export "${CLOUDIATR_BPOOL_NAME}" || true + zpool export "${CLOUDIATR_RPOOL_NAME}" || true + umount "${CLOUDIATR_NEW_SYSTEM_DIR}" || true + geli kill -a || true + fi +} + +cloudiatr_request_consent() { + local message="${*}" \ + response + + if "${CLOUDIATR_DONT_ASK_JUST_KISS}"; then + # ... the data goodbye. + return 0 + fi + + echo -n "cloudiatr: ${message} [y/n] " + # XXX: Don't use "read -p" as it may work unreliably + read response + + # "No" means "no". Everything but "y" also means "no". + [ "${response}" = "y" ] +} + +cloudiatr_has_eviction_consent() { + echo "cloudiatr (${CLOUDIATR_VERSION}) can't wait to evict '$(hostname)' ..." + echo + echo "Depending on your jurisdiction, 'eviction without consent' may be against the law." + echo "cloudiatr doesn't bother to make backups of the existing data. That's what zogftw is for." + echo + cloudiatr_request_consent "Continue eviction?" +} + +cloudiatr_evict() { + if cloudiatr_has_eviction_consent; then + if "${CLOUDIATR_MURDER_DEATH_KILL_REQUESTED}"; then + cloudiatr_murder_death_kill + else + cloudiatr_fyi "You asked for it ..." + fi + cloudiatr_fyi "Eviction in progress ..." + cloudiatr_evict_local_system + cloudiatr_fyi "Looks like somebody managed to install a real operating system ..." + if cloudiatr_request_consent "Reboot now?"; then + shutdown -r now + fi + return 0 + else + cloudiatr_fyi "Eviction aborted in time ..." + return 1 + fi +} + +cloudiatr_has_soft_protect_consent() { + cloudiatr_request_consent "Put $(uname) in 'Soviet Germany' mode?" +} + +# XXX: Only works for the cloudiatr disk layout. +cloudiatr_soft_protect() { + local \ + mirror_name device_to_clear number_of_disks + + mirror_name="vdev-remains" + number_of_disks="$(cloudiatr_get_number_of_disks)" + + cloudiatr_fyi "Destroying ${CLOUDIATR_BPOOL_NAME} ..." + cloudiatr_fyi "Use 'geli kill -a' to 'hard-protect' your data right now. No recovery without remote backups!" + + if zpool list "${CLOUDIATR_BPOOL_NAME}" >/dev/null 2>&1; then + zpool destroy "${CLOUDIATR_BPOOL_NAME}" || true + fi + + if [ "${number_of_disks}" = 1 ]; then + cloudiatr_fyi "Nuking former ${CLOUDIATR_BPOOL_NAME} vdev from orbit ..." + device_to_clear="$(cloudiatr_get_bpool_geoms)" + else + cloudiatr_fyi "Nuking former ${CLOUDIATR_BPOOL_NAME} vdevs from orbit using gmirror power ..." + gmirror load 2>/dev/null || true + gmirror label "${mirror_name}" $(cloudiatr_get_bpool_geoms) + device_to_clear="/dev/mirror/${mirror_name}" + fi + + geli onetime -s 4096 -e "${CLOUDIATR_GELI_EALGO}" "${device_to_clear}" + dd if=/dev/zero bs=1M of="${device_to_clear}.eli" 2>/dev/null || true + geli detach "${device_to_clear##/dev/}" + + cloudiatr_fyi "Done. Levelling nuked wasteland with zeroes ..." + dd if=/dev/zero bs=1M of="${device_to_clear}" 2>/dev/null || true + + if [ "${number_of_disks}" != 1 ]; then + gmirror destroy "${mirror_name}" + fi + cloudiatr_fyi "Done. $(uname) should remain working as expected until the next shutdown ..." + cloudiatr_fyi "Remember to 'unprotect' the system before consensual reboots (or use the opportunity to test your backup system)" +} + +cloudiatr_check_privileges() { + local \ + uid user + + uid="$(id -u)" + user="$(id -un)" + + if [ "${uid}" != 0 ]; then + cloudiatr_wtf "Check your privileges, $user. It looks like you might not have enough of them!" + cloudiatr_fyi "Hint: Using 'sudo' or 'su' might help." + return 1 + fi + + return 0 +} + +cloudiatr_usage() { + local \ + subcommand + + for subcommand in clean-up cmd recreate-bpool evict soft-protect show-config; do + echo "cloudiatr [-f config-file] $subcommand" + done +} + +cloudiatr_update_base() { + local \ + base_dist base_dbg_dist + + base_dist="${CLOUDIATR_DIST_DIR}/base.txz" + base_dbg_dist="${CLOUDIATR_DIST_DIR}/base-dbg.txz" + + if [ ! -f "${base_dist}" ]; then + cloudiatr_wtf "Base update impossible. '${base_dist}' does not exist" + return 1 + fi + cloudiatr_wtf "Updating base system ..." + chflags -R noschg /bin/ /lib/ /sbin/ /usr/bin/ /usr/lib /libexec/ || return 1 + cd / || return 1 + tar xpf "${base_dist}" \ + --exclude ./etc/ --exclude ./var/empty --exclude ./usr/src || return 1 + if [ -f "${base_dbg_dist}" ]; then + tar xpf "${base_dbg_dist}" || return 1 + fi +} + +cloudiatr_main() { + local mode + + if [ "${1}" = "-f" ]; then + shift + CLOUDIATR_CONFIG_FILE="${1}" + shift + if [ -z "${CLOUDIATR_CONFIG_FILE}" ] || + ! [ -f "${CLOUDIATR_CONFIG_FILE}" ]; then + cloudiatr_wtf "No existing config file specified." + return 1 + fi + fi + + mode="${1}" + + if [ "${mode}" != "cmd" ] && [ "${mode}" != "show-config" ]; then + cloudiatr_check_privileges || return 1 + fi + + cloudiatr_init "${mode}" + + case "${mode}" in + clean-up) + set +e + cloudiatr_clean_up + ;; + cmd) + shift + "${@}" + ;; + recreate-bpool) + cloudiatr_create_bpool + cloudiatr_populate_bpool "/" + ;; + evict) + cloudiatr_evict + ;; + soft-protect) + if cloudiatr_has_soft_protect_consent; then + cloudiatr_soft_protect + fi + ;; + update) + cloudiatr_update_base || return 1 + ;; + show-config) + cloudiatr_show_config + ;; + #rekey) + # XXX: Not yet implemented + *) + cloudiatr_usage + cloudiatr_wtf "Invalid or missing subcommand" + return 1 + ;; + esac + +} + +cloudiatr_main "${@}" -- 2.11.0 From b3f52c559d2ddaf2319933a5e637db594a89beff Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 30 Dec 2016 16:54:28 +0100 Subject: [PATCH 243/257] g_dev_orphan(): Return early if the device is already gone Supposed to fix a panic that could occur while running "cdcontrol eject" after using the physical ejection key on the device: Unread portion of the kernel message buffer: stack pointer = 0x28:0xfffffe01eba0a9e0 frame pointer = 0x28:0xfffffe01eba0a9f0 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, long 1, def32 0, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 13 (g_event) trap number = 12 panic: page fault cpuid = 3 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe01eba0a560 vpanic() at vpanic+0x182/frame 0xfffffe01eba0a5e0 panic() at panic+0x43/frame 0xfffffe01eba0a640 trap_fatal() at trap_fatal+0x331/frame 0xfffffe01eba0a6a0 trap_pfault() at trap_pfault+0x1e3/frame 0xfffffe01eba0a700 trap() at trap+0x273/frame 0xfffffe01eba0a910 calltrap() at calltrap+0x8/frame 0xfffffe01eba0a910 --- trap 0xc, rip = 0xffffffff80500fde, rsp = 0xfffffe01eba0a9e0, rbp = 0xfffffe01eba0a9f0 --- g_dev_orphan() at g_dev_orphan+0x2e/frame 0xfffffe01eba0a9f0 g_resize_provider_event() at g_resize_provider_event+0x71/frame 0xfffffe01eba0aa20 g_run_events() at g_run_events+0x20e/frame 0xfffffe01eba0aa70 fork_exit() at fork_exit+0x85/frame 0xfffffe01eba0aab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01eba0aab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Uptime: 3h17m41s Dumping 1120 out of 8055 MB:..2%..12%..22%..32%..42%..52%..62%..72%..82%..92% Reading symbols from /usr/lib/debug/boot/kernel/zfs.ko.debug...done. [...] Loaded symbols for /usr/lib/debug/boot/kernel/iicbb.ko.debug #0 doadump (textdump=1) at pcpu.h:222 222 pcpu.h: No such file or directory. in pcpu.h (kgdb) where #0 doadump (textdump=1) at pcpu.h:222 #1 0xffffffff805cce3e in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:366 #2 0xffffffff805cd40b in vpanic (fmt=, ap=) at /usr/src/sys/kern/kern_shutdown.c:759 #3 0xffffffff805cd243 in panic (fmt=0x0) at /usr/src/sys/kern/kern_shutdown.c:690 #4 0xffffffff808eece1 in trap_fatal (frame=0xfffffe01eba0a920, eva=8) at /usr/src/sys/amd64/amd64/trap.c:801 #5 0xffffffff808eeed3 in trap_pfault (frame=0xfffffe01eba0a920, usermode=0) at /usr/src/sys/amd64/amd64/trap.c:658 #6 0xffffffff808ee4d3 in trap (frame=0xfffffe01eba0a920) at /usr/src/sys/amd64/amd64/trap.c:421 #7 0xffffffff808d2701 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:236 #8 0xffffffff80500fde in g_dev_orphan (cp=0xfffff800069e6280) at /usr/src/sys/geom/geom_dev.c:754 #9 0xffffffff80509ff1 in g_resize_provider_event (arg=, flag=) at /usr/src/sys/geom/geom_subr.c:631 #10 0xffffffff80504f1e in g_run_events () at /usr/src/sys/geom/geom_event.c:264 #11 0xffffffff805830e5 in fork_exit (callout=0xffffffff805079c0 , arg=0x0, frame=0xfffffe01eba0aac0) at /usr/src/sys/kern/kern_fork.c:1040 #12 0xffffffff808d2c3e in fork_trampoline () at /usr/src/sys/amd64/amd64/exception.S:611 #13 0x0000000000000000 in ?? () Current language: auto; currently minimal (kgdb) f 8 #8 0xffffffff80500fde in g_dev_orphan (cp=0xfffff800069e6280) at /usr/src/sys/geom/geom_dev.c:754 754 g_trace(G_T_TOPOLOGY, "g_dev_orphan(%p(%s))", cp, cp->geom->name); (kgdb) p cp $1 = (struct g_consumer *) 0xfffff800069e6280 (kgdb) p cp->geom $2 = (struct g_geom *) 0xfffff801acd6d100 (kgdb) p cp->geom->name $3 = 0xfffff8007479bf60 "cd0" (kgdb) f 8 #8 0xffffffff80500fde in g_dev_orphan (cp=0xfffff800069e6280) at /usr/src/sys/geom/geom_dev.c:754 754 g_trace(G_T_TOPOLOGY, "g_dev_orphan(%p(%s))", cp, cp->geom->name); (kgdb) l 754 g_trace(G_T_TOPOLOGY, "g_dev_orphan(%p(%s))", cp, cp->geom->name); 755 756 /* Reset any dump-area set on this device */ 757 if (dev->si_flags & SI_DUMPDEV) 758 (void)set_dumper(NULL, NULL, curthread); 759 760 /* Destroy the struct cdev *so we get no more requests */ 761 destroy_dev_sched_cb(dev, g_dev_callback, cp); 762 } 763 (kgdb) p dev->si_flags Cannot access memory at address 0x8 (kgdb) p dev $4 = (struct cdev *) 0x0 Last message before the panic: Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:23 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:24 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:26 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c0 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 39 c6 c1 00 00 01 00 Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: ILLEGAL REQUEST asc:6f,3 (Read of scrambled sector without authentication) Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:09:33 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:33:49 t520 kernel: ahcich1: Timeout on slot 8 port 0 Dec 30 16:33:49 t520 kernel: ahcich1: is 00000000 cs 00000100 ss 00000000 rs 00000100 tfd d0 serr 00000000 cmd 000cc817 Dec 30 16:33:49 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 eb 00 00 01 00 Dec 30 16:33:49 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: Command timeout Dec 30 16:33:49 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 eb 00 00 01 00 Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: MEDIUM ERROR asc:10,0 (ID CRC or ECC error) Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:33:57 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:34:05 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 ad 00 00 04 00 Dec 30 16:34:05 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:05 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:05 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: MEDIUM ERROR asc:10,0 (ID CRC or ECC error) Dec 30 16:34:05 t520 kernel: (cd0:ahcich1:0:0:0): Retrying command (per sense data) Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 ad 00 00 04 00 Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: MEDIUM ERROR asc:10,0 (ID CRC or ECC error) Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): Error 5, Retries exhausted Dec 30 16:34:14 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x5 back Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 eb 00 00 01 00 Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: NOT READY asc:3a,0 (Medium not present) Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): Error 6, Unretryable error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x6 back Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 ea 00 00 01 00 Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: NOT READY asc:3a,0 (Medium not present) Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): Error 6, Unretryable error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x6 back Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 ea 00 00 01 00 Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: NOT READY asc:3a,0 (Medium not present) Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): Error 6, Unretryable error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x6 back Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): READ(10). CDB: 28 00 00 3e a2 eb 00 00 01 00 Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): CAM status: SCSI Status Error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI status: Check Condition Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): SCSI sense: NOT READY asc:3a,0 (Medium not present) Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): Error 6, Unretryable error Dec 30 16:34:18 t520 kernel: (cd0:ahcich1:0:0:0): cddone: got error 0x6 back PR: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=215856 PR submission date: 2017-01-07 Obtained from: ElectroBSD --- sys/geom/geom_dev.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sys/geom/geom_dev.c b/sys/geom/geom_dev.c index d74b92e59776..543df1db3614 100644 --- a/sys/geom/geom_dev.c +++ b/sys/geom/geom_dev.c @@ -753,6 +753,9 @@ g_dev_orphan(struct g_consumer *cp) dev = sc->sc_dev; g_trace(G_T_TOPOLOGY, "g_dev_orphan(%p(%s))", cp, cp->geom->name); + if (dev == NULL) + return; + /* Reset any dump-area set on this device */ if (dev->si_flags & SI_DUMPDEV) (void)set_dumper(NULL, NULL, curthread); -- 2.11.0 From bc62b6ac8d001a3c8f025e3d788a94b891713bba Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 15 Jan 2017 11:38:02 +0100 Subject: [PATCH 244/257] release: As an experiment, use the OSRELEASE verbatim as VOLUME_LABEL Obtained from: ElectroBSD --- release/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/release/Makefile b/release/Makefile index 48c7475c23b5..4111d18ddf0a 100644 --- a/release/Makefile +++ b/release/Makefile @@ -79,11 +79,10 @@ ${_V}!= eval $$(awk '/^${_V}=/{print}' ${.CURDIR}/../sys/conf/newvers.sh); echo .for _V in ${TARGET_ARCH} .if !empty(TARGET:M${_V}) OSRELEASE= ${TYPE}-${REVISION}-${BRANCH}-${TARGET} -VOLUME_LABEL= ${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-]/_/g}_${TARGET} .else OSRELEASE= ${TYPE}-${REVISION}-${BRANCH}-${TARGET}-${TARGET_ARCH} -VOLUME_LABEL= ${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-]/_/g}_${TARGET_ARCH} .endif +VOLUME_LABEL= ${OSRELEASE} .endfor .endif -- 2.11.0 From 8168608c3c6d08ccbf7836a817e629be069f5132 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 11 Dec 2016 16:15:58 +0100 Subject: [PATCH 245/257] Import reproduce-electrobsd.sh 2017-01-16-4076de35031 Obtained from: ElectroBSD --- reproduce-electrobsd.sh | 142 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100755 reproduce-electrobsd.sh diff --git a/reproduce-electrobsd.sh b/reproduce-electrobsd.sh new file mode 100755 index 000000000000..cff20085a013 --- /dev/null +++ b/reproduce-electrobsd.sh @@ -0,0 +1,142 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2016 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## + +# reproduce-electrobsd.sh +# +# This script builds ElectroBSD in a jail, updates the jail +# using the built binaries and builds ElectroBSD again. +# +# The second build should thus always result with the same +# binaries, even if different ElectroBSD versions were used +# for the first build, or (not yet tested) if the first build +# was done with FreeBSD binaries. +# +# The following steps are not yet automated: +# +# 1) Create an /etc/jail.conf with a section like this one: +# +# ElectroBSD-amd64 { +# host.hostname=ElectroBSD-amd64; +# persist; +# children.max=0; +# allow.mount; +# allow.mount.devfs; +# enforce_statfs=1; +# path=/usr/jails/ElectroBSD-amd64; +# exec.start="mount -t devfs devfs /dev"; +# exec.stop="umount /dev"; +# } +# +# 2) Create two new ZFS datasets for the build jail: +# +# /usr/jails/ElectroBSD-amd64 +# /usr/jails/ElectroBSD-amd64/usr +# +# 3) Extract a base.txz in it. +# +# 4) Mount an UFS file system in "/usr/jails/ElectroBSD-amd64/usr/obj" +# and allow the BUILD_USER to run it. This step is only required +# to get reproducible images, the tarballs should be reproducible +# when using ZFS as well. +# +# 5) Create a dedicated /usr/src dataset on the host and populate it +# with src.txz +# +# 6) Install sudo and zogftw on the host. + +HOST_SRC_DIR="/usr/src" +HOST_BUILDLOG_DIR="/var/log/buildlogs" + +BUILD_JAIL="${BUILD_JAIL-ElectroBSD-amd64}" +BUILD_JAIL_ROOT="/usr/jails/${BUILD_JAIL}" +BUILD_JAIL_SRC_DIR="${BUILD_JAIL_ROOT}/usr/src" +BUILD_USER="$(id -un)" +BUILD_NICE_VALUE=20 +BUILD_CORE_COUNT=2 +BUILD_TWICE="${BUILD_TWICE-true}" +# This is relative to the BUILD_JAIL_ROOT +BUILD_DISTFILE_DIR_PREFIX=/usr/obj/usr/src/ + +prepare_build_jail() { + local \ + src_dataset last_src_snapshot src_clone build_jail_dataset + + # Use vanilla zogftw configuration without potentionally + # existing fancy custom hooks that could slow use down. + export ZOGFTW_CONFIG_FILE='' + + zogftw snap /usr/src + + src_dataset=$(zogftw zcmd get_dataset_from_path "${HOST_SRC_DIR}") + if [ -z "${src_dataset}" ]; then + echo "Failed to get src dataset" + return 1 + fi + src_clone=$(zogftw zcmd get_dataset_from_path "${BUILD_JAIL_SRC_DIR}") + if [ -n "${src_clone}" ]; then + # The build jail already has a clone /usr/src, + # delete it as it may be stale + sudo zfs destroy "${src_clone}" || return 1 + else + build_jail_dataset=$(zogftw zcmd get_dataset_from_path "${BUILD_JAIL_ROOT}") + if [ -z "${build_jail_dataset}" ]; then + echo "Failed to get root jail dataset" + return 1 + fi + src_clone="${build_jail_dataset}/usr/src" + echo "Will create fresh clone in $src_clone" + fi + last_src_snapshot=$(zogftw zcmd get_last_snapshot "${src_dataset}") + if [ -z "${last_src_snapshot}" ]; then + echo "Failed to get src clone" + return 1 + fi + sudo zfs clone "${last_src_snapshot}" "${src_clone}" || return 1 +} + +reproduce() { + local \ + timestamp logfile build distfile_dir_prefix + + timestamp=$(date "+%Y-%m-%d_%H:%M") + logfile="${HOST_BUILDLOG_DIR}/reprolog-${timestamp}" + build="$(grep ^BUILD= /usr/src/reproduce.conf | cut -d = -f 2)" + distfile_dir_prefix="/usr/obj/usr/src/$build" + + script "${logfile}" \ + sudo nice -n "${BUILD_NICE_VALUE}" \ + sh -c "jail -c '${BUILD_JAIL}' && \ + jexec -u ${BUILD_USER} '${BUILD_JAIL}' \ + /usr/src/reproduce.sh -j${BUILD_CORE_COUNT} -d ${distfile_dir_prefix}-j1 -a && \ + jexec '${BUILD_JAIL}' make -C /usr/src installworld NO_FSCHG='yes' && \ + ${BUILD_TWICE} && jexec -u ${BUILD_USER} '${BUILD_JAIL}' \ + /usr/src/reproduce.sh -j${BUILD_CORE_COUNT} -d ${distfile_dir_prefix}-j2; + jail -r '${BUILD_JAIL}'" || return 1 + + # Compare checksums. Use a separate script call to make + # sure the checksums from the second run are flushed to + # the file + script -a "${logfile}" sh -c "grep ^SHA256 '${logfile}' | sort -k 4 | column -t" +} + +main() { + prepare_build_jail || return 1 + reproduce || return 1 +} + +main -- 2.11.0 From 92ca151c3367c021a28ada2dbc24b474c13725b1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 17:26:59 +0200 Subject: [PATCH 246/257] Add reproduce.sh which makes reproducing ElectroBSD more convenient Squashed commits worth mentioning: - Add -j option to overwrite the maximum number of make jobs - Assert that the source directory is untainted (according to strip-freebsd.sh) and add -a flag to remove offending files - Allow to resume a build by using the -r flag. Obtained from: ElectroBSD --- reproduce.sh | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100755 reproduce.sh diff --git a/reproduce.sh b/reproduce.sh new file mode 100755 index 000000000000..d7b4fa941817 --- /dev/null +++ b/reproduce.sh @@ -0,0 +1,196 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## + +# reproduce.sh +# +# Script to make reproducing an ElectroBSD build more convenient. +# Before using it, make sure BUILD and EPOCH contain values other +# than __BUILD__ and __EPOCH__, either by editing the script or +# by putting them in a configuration file that is speficied with +# the -f option. + +# These variables have to be set to the values used for the build +# that is supposed to be reproduced. ${SRC_DIR} must contain the +# matching sources! +BUILD=__BUILD__ +EPOCH=__EPOCH__ + +# This is just a suggestion, feel free to overwrite it with the -j option. +MAX_MAKE_JOBS="${MAX_MAKE_JOBS-4}" + +# Currently hardcoded. +SRC_DIR=/usr/src +OPTIONAL_CONFIG_FILE="${SRC_DIR}/reproduce.conf" + +# Make sure we respawn with the same script, even if it is located +# outside the SRC_DIR and called with a relative path. +REPRODUCE_SH="$(realpath "${0}")" + +# When set to true, existing object files will be reused. +# If the source files changed, the result will not be reproducible! +RESUME_BUILD="${RESUME_BUILD-false}" + +announce_status() { + local msg \ + timestamp + + msg="${*}" + timestamp=$(date "+%Y-%m-%d %H:%M") + + echo "${timestamp}: ${msg}" +} + +reproduce_all_the_things() { + + if "${RESUME_BUILD}"; then + announce_status "Resuming ..." + export KERNFAST=1 + export NO_CLEAN=1 + fi + + announce_status "Starting to build the kernel" + make buildkernel || return 1 + + announce_status "Starting to build the world" + make -j${MAX_MAKE_JOBS} buildworld || return 1 + + # Make sure obj files aren't dumped in ${SRC_DIR} + mkdir -p "/usr/obj${SRC_DIR}/release" || return 1 + + if ! "${RESUME_BUILD}"; then + announce_status "Starting to clean the release dir" + make -C "${SRC_DIR}/release" clean + fi + announce_status "Starting to build the release" + time make -C "${SRC_DIR}/release" memstick NO_FSCHG="yes" || return 1 + + announce_status "Done with release memstick for ${REPRO_SEED}" +} + +assert_untainted_source_tree() { + local auto_untaint \ + untaint_commands + + auto_untaint="${1}" + + untaint_commands="$(sh ./release/scripts/strip-freebsd.sh .)" + if [ -n "${untaint_commands}" ] ; then + if $auto_untaint; then + echo "Auto untainting $(pwd)" + echo "${untaint_commands}" | sh -x || return 1 + else + echo "${SRC_DIR} is tainted. Use -a flag to auto-untaint it." + return 1 + fi + fi + return 0 +} + +respawn_with_clean_environment() { + exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ + LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ + MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ + REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" +} + +main() { + local args \ + auto_untaint config_file fake_user dry_run + + fake_user=elektropunker + + auto_untaint=false + dry_run=false + config_file="${OPTIONAL_CONFIG_FILE}" + + args=$(getopt af:j:nr $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -a) + shift + auto_untaint="true" + ;; + -j) + shift; + MAX_MAKE_JOBS="${1}" + shift; + ;; + -f) + shift; + config_file="${1}" + shift; + if [ ! -f "${config_file}" ]; then + echo "Config file ${config_file} does not exist" + exit 2 + fi + ;; + -n) + dry_run=true + shift + ;; + -r) + shift + RESUME_BUILD=true + ;; + --) + shift; break + ;; + esac + done + + if [ -f "${config_file}" ]; then + announce_status "Reading config from ${config_file}" + . "${config_file}" || exit 2 + fi + + if [ "${BUILD}" = "__BUILD__" ]; then + announce_status "BUILD not set" + return 1 + fi + if [ "${EPOCH}" = "__EPOCH__" ]; then + announce_status "EPOCH not set" + return 1 + fi + if [ -n "${ALREADY_RESPAWNED}" -a "${ALREADY_RESPAWNED}" = 1 ]; then + if [ -z "${REPRO_SEED}" ]; then + announce_status "Respawned with REPRO_SEED unset" + return 1 + fi + reproduce_all_the_things + return + fi + + export REPRO_SEED="${fake_user}:${BUILD}:${EPOCH}" + + announce_status "REPRO_SEED=${REPRO_SEED}" + + cd "${SRC_DIR}" || return 1 + + assert_untainted_source_tree "${auto_untaint}" || return 1 + + if ! $dry_run; then + respawn_with_clean_environment + fi +} + +main "${@}" -- 2.11.0 From baa17218216eff0fd83827cb202cf2ac1bb90e8e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 25 Dec 2015 15:24:18 +0100 Subject: [PATCH 247/257] reproduce.sh: Add -p option to change the prefix for the source and object directories This could be useful when building ElectroBSD as port. Obtained from: ElectroBSD --- reproduce.sh | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index d7b4fa941817..31be55b7ba04 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -33,8 +33,14 @@ EPOCH=__EPOCH__ # This is just a suggestion, feel free to overwrite it with the -j option. MAX_MAKE_JOBS="${MAX_MAKE_JOBS-4}" -# Currently hardcoded. -SRC_DIR=/usr/src +# Prefix to use for SRC_DIR and MAKEOBJDIRPREFIX +DIRECTORY_PREFIX="${DIRECTORY_PREFIX-/}" + +# Currently somewhat hardcoded. +SRC_DIR="${DIRECTORY_PREFIX}usr/src" +MAKEOBJDIRPREFIX="${DIRECTORY_PREFIX}usr/obj" + +# Config file location when -f isn't specified OPTIONAL_CONFIG_FILE="${SRC_DIR}/reproduce.conf" # Make sure we respawn with the same script, even if it is located @@ -62,6 +68,8 @@ reproduce_all_the_things() { export KERNFAST=1 export NO_CLEAN=1 fi + export MAKEOBJDIRPREFIX + announce_status "MAKEOBJDIRPREFIX is set to ${MAKEOBJDIRPREFIX}" announce_status "Starting to build the kernel" make buildkernel || return 1 @@ -70,7 +78,7 @@ reproduce_all_the_things() { make -j${MAX_MAKE_JOBS} buildworld || return 1 # Make sure obj files aren't dumped in ${SRC_DIR} - mkdir -p "/usr/obj${SRC_DIR}/release" || return 1 + mkdir -p "${MAKEOBJDIRPREFIX}${SRC_DIR}/release" || return 1 if ! "${RESUME_BUILD}"; then announce_status "Starting to clean the release dir" @@ -105,6 +113,7 @@ respawn_with_clean_environment() { exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ + DIRECTORY_PREFIX="${DIRECTORY_PREFIX}" \ REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" } @@ -118,7 +127,7 @@ main() { dry_run=false config_file="${OPTIONAL_CONFIG_FILE}" - args=$(getopt af:j:nr $*) + args=$(getopt af:j:npr $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -148,6 +157,17 @@ main() { dry_run=true shift ;; + -p) + shift + DIRECTORY_PREFIX="${1}" + shift; + if [ ! -d "${DIRECTORY_PREFIX}" ]; then + echo "Directory ${DIRECTORY_PREFIX} specified with -p does not exist" + exit 2 + fi + # This is only needed for the cd below + SRC_DIR="${DIRECTORY_PREFIX}${SRC_DIR}" + ;; -r) shift RESUME_BUILD=true -- 2.11.0 From 42a3a1e2cb24247e717d09c2abf06d2df5efb6df Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 Jan 2016 19:57:05 +0100 Subject: [PATCH 248/257] reproduce.sh: Build kernel after world to make sure the toolchain is fresh Obtained from: ElectroBSD --- reproduce.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index 31be55b7ba04..4783dd510775 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -71,12 +71,14 @@ reproduce_all_the_things() { export MAKEOBJDIRPREFIX announce_status "MAKEOBJDIRPREFIX is set to ${MAKEOBJDIRPREFIX}" - announce_status "Starting to build the kernel" - make buildkernel || return 1 - + # We build the world first, so the kernel is built + # with a freshly built toolchain. announce_status "Starting to build the world" make -j${MAX_MAKE_JOBS} buildworld || return 1 + announce_status "Starting to build the kernel" + make buildkernel || return 1 + # Make sure obj files aren't dumped in ${SRC_DIR} mkdir -p "${MAKEOBJDIRPREFIX}${SRC_DIR}/release" || return 1 -- 2.11.0 From 1f546a8c185b8a7c87d8912975fd4592e957a646 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 15 Jan 2016 18:35:29 +0100 Subject: [PATCH 249/257] reproduce.sh: Move created files to their own directory when done Obtained from: ElectroBSD --- reproduce.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/reproduce.sh b/reproduce.sh index 4783dd510775..125d9fa407a1 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -62,7 +62,10 @@ announce_status() { } reproduce_all_the_things() { + local \ + f release_dir final_dir + release_dir="${MAKEOBJDIRPREFIX}${SRC_DIR}/release" if "${RESUME_BUILD}"; then announce_status "Resuming ..." export KERNFAST=1 @@ -80,7 +83,7 @@ reproduce_all_the_things() { make buildkernel || return 1 # Make sure obj files aren't dumped in ${SRC_DIR} - mkdir -p "${MAKEOBJDIRPREFIX}${SRC_DIR}/release" || return 1 + mkdir -p "${release_dir}" || return 1 if ! "${RESUME_BUILD}"; then announce_status "Starting to clean the release dir" @@ -90,6 +93,18 @@ reproduce_all_the_things() { time make -C "${SRC_DIR}/release" memstick NO_FSCHG="yes" || return 1 announce_status "Done with release memstick for ${REPRO_SEED}" + + cd "${release_dir}" || return 1 + + final_dir="${MAKEOBJDIRPREFIX}${SRC_DIR}/${BUILD}-$(date "+%Y-%m-%d-%H:%M")" + mkdir "${final_dir}" || return 1 + for f in *.txz MANIFEST; do + mv "${f}" "${final_dir}/" || return + done + mv memstick.img "${final_dir}/${BUILD}.img" || return 1 + + announce_status "Created files copied to ${final_dir}" + sha256 "${final_dir}"/* } assert_untainted_source_tree() { -- 2.11.0 From 3c048ec876c0965a97af77ac1fa2f6aca6d9123c Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 Jul 2016 12:19:53 +0200 Subject: [PATCH 250/257] reproduce.sh: Hardcode the number of xz threads to 2 Due to some upstream change hardcoding it to something is required now to get reproducible results when using systems with a different number of cores. Obtained from: ElectroBSD --- reproduce.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index 125d9fa407a1..6eb83f54c451 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -51,6 +51,13 @@ REPRODUCE_SH="$(realpath "${0}")" # If the source files changed, the result will not be reproducible! RESUME_BUILD="${RESUME_BUILD-false}" +# Number of threads to use when compressing with xz. +# +# The upstream default is 0 (auto-tune) which results +# in unreproducible results when using systems with a +# different number of cores. +XZ_THREADS="${XZ_THREADS-2}" + announce_status() { local msg \ timestamp @@ -90,7 +97,8 @@ reproduce_all_the_things() { make -C "${SRC_DIR}/release" clean fi announce_status "Starting to build the release" - time make -C "${SRC_DIR}/release" memstick NO_FSCHG="yes" || return 1 + time make -C "${SRC_DIR}/release" \ + memstick XZ_THREADS=${XZ_THREADS} NO_FSCHG="yes" || return 1 announce_status "Done with release memstick for ${REPRO_SEED}" @@ -130,7 +138,7 @@ respawn_with_clean_environment() { exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ - DIRECTORY_PREFIX="${DIRECTORY_PREFIX}" \ + DIRECTORY_PREFIX="${DIRECTORY_PREFIX}" XZ_THREADS=${XZ_THREADS} \ REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" } -- 2.11.0 From e3e98c067e78d7d2290063eb6c70db070c220bb5 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Nov 2016 15:13:42 +0100 Subject: [PATCH 251/257] reproduce.sh: Add a -d option to specifiy the directory to move the produced distfiles into Obtained from: ElectroBSD --- reproduce.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index 6eb83f54c451..cd5c5cb0a30b 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -1,7 +1,7 @@ #!/bin/sh ########################################################################## -# Copyright (c) 2015 Fabian Keil +# Copyright (c) 2015-2016 Fabian Keil # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above @@ -70,7 +70,7 @@ announce_status() { reproduce_all_the_things() { local \ - f release_dir final_dir + f release_dir release_dir="${MAKEOBJDIRPREFIX}${SRC_DIR}/release" if "${RESUME_BUILD}"; then @@ -104,15 +104,14 @@ reproduce_all_the_things() { cd "${release_dir}" || return 1 - final_dir="${MAKEOBJDIRPREFIX}${SRC_DIR}/${BUILD}-$(date "+%Y-%m-%d-%H:%M")" - mkdir "${final_dir}" || return 1 + mkdir -p "${DISTFILE_DIR}" || return 1 for f in *.txz MANIFEST; do - mv "${f}" "${final_dir}/" || return + mv "${f}" "${DISTFILE_DIR}/" || return done - mv memstick.img "${final_dir}/${BUILD}.img" || return 1 + mv memstick.img "${DISTFILE_DIR}/${BUILD}.img" || return 1 - announce_status "Created files copied to ${final_dir}" - sha256 "${final_dir}"/* + announce_status "Created files copied to ${DISTFILE_DIR}" + sha256 "${DISTFILE_DIR}"/* } assert_untainted_source_tree() { @@ -137,6 +136,7 @@ assert_untainted_source_tree() { respawn_with_clean_environment() { exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ + DISTFILE_DIR="${DISTFILE_DIR}" \ MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ DIRECTORY_PREFIX="${DIRECTORY_PREFIX}" XZ_THREADS=${XZ_THREADS} \ REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" @@ -152,7 +152,7 @@ main() { dry_run=false config_file="${OPTIONAL_CONFIG_FILE}" - args=$(getopt af:j:npr $*) + args=$(getopt ad:f:j:npr $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -164,6 +164,11 @@ main() { shift auto_untaint="true" ;; + -d) + shift; + DISTFILE_DIR="${1}" + shift; + ;; -j) shift; MAX_MAKE_JOBS="${1}" @@ -226,6 +231,7 @@ main() { fi export REPRO_SEED="${fake_user}:${BUILD}:${EPOCH}" + export DISTFILE_DIR="${DISTFILE_DIR-${MAKEOBJDIRPREFIX}${SRC_DIR}/${BUILD}-$(date +%Y-%m-%d-%H:%M)}" announce_status "REPRO_SEED=${REPRO_SEED}" -- 2.11.0 From 7227681155df2c05705fdbcb56dacb82f93a7ecc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 16 Jan 2017 18:04:45 +0100 Subject: [PATCH 252/257] reproduce.sh: time buildworld and buildkernel as well, consistently use -l. Squash Obtained from: ElectroBSD --- reproduce.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/reproduce.sh b/reproduce.sh index cd5c5cb0a30b..325b28e21d8f 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -84,10 +84,10 @@ reproduce_all_the_things() { # We build the world first, so the kernel is built # with a freshly built toolchain. announce_status "Starting to build the world" - make -j${MAX_MAKE_JOBS} buildworld || return 1 + time -l make -j${MAX_MAKE_JOBS} buildworld || return 1 announce_status "Starting to build the kernel" - make buildkernel || return 1 + time -l make buildkernel || return 1 # Make sure obj files aren't dumped in ${SRC_DIR} mkdir -p "${release_dir}" || return 1 @@ -97,8 +97,8 @@ reproduce_all_the_things() { make -C "${SRC_DIR}/release" clean fi announce_status "Starting to build the release" - time make -C "${SRC_DIR}/release" \ - memstick XZ_THREADS=${XZ_THREADS} NO_FSCHG="yes" || return 1 + time -l make -C "${SRC_DIR}/release" \ + memstick XZ_THREADS=${XZ_THREADS} NO_FSCHG="yes" || return 1 announce_status "Done with release memstick for ${REPRO_SEED}" -- 2.11.0 From 568f2770b6b3af07c63f42f0c6e2b7142ee1ab74 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 17 Mar 2016 12:51:02 +0100 Subject: [PATCH 253/257] sys/cam: Retry in case of 'uncorrectable' errors At least in case of the LITE-ON DVDRW SOHW-1693S with firmware KC4B, they are recoverable. Maybe this should be done as device-specific quirk. Obtained from: ElectroBSD --- sys/cam/scsi/scsi_all.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/cam/scsi/scsi_all.c b/sys/cam/scsi/scsi_all.c index 6732243f84cb..83ea7ea9bf0c 100644 --- a/sys/cam/scsi/scsi_all.c +++ b/sys/cam/scsi/scsi_all.c @@ -1407,8 +1407,8 @@ static struct asc_table_entry asc_table[] = { { SST(0x11, 0x04, SS_FATAL|EIO, "Unrecovered read error - auto reallocate failed") }, /* WRO B */ - { SST(0x11, 0x05, SS_FATAL|EIO, - "L-EC uncorrectable error") }, + { SST(0x11, 0x05, SS_RDEF, + "L-EC 'uncorrectable' error") }, /* WRO B */ { SST(0x11, 0x06, SS_FATAL|EIO, "CIRC unrecovered error") }, -- 2.11.0 From 6cb73603b3253ec2aaad1b83f7fe23835060a828 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 20 Jan 2017 15:31:18 +0100 Subject: [PATCH 254/257] Revert "sys/cam: Retry in case of 'uncorrectable' errors" Suspected to prevent ripping of "Puhdys - Undercover". Obtained from: ElectroBSD --- sys/cam/scsi/scsi_all.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/cam/scsi/scsi_all.c b/sys/cam/scsi/scsi_all.c index 83ea7ea9bf0c..6732243f84cb 100644 --- a/sys/cam/scsi/scsi_all.c +++ b/sys/cam/scsi/scsi_all.c @@ -1407,8 +1407,8 @@ static struct asc_table_entry asc_table[] = { { SST(0x11, 0x04, SS_FATAL|EIO, "Unrecovered read error - auto reallocate failed") }, /* WRO B */ - { SST(0x11, 0x05, SS_RDEF, - "L-EC 'uncorrectable' error") }, + { SST(0x11, 0x05, SS_FATAL|EIO, + "L-EC uncorrectable error") }, /* WRO B */ { SST(0x11, 0x06, SS_FATAL|EIO, "CIRC unrecovered error") }, -- 2.11.0 From 8dec87d1add23c492ca07706de4eb392a7c75826 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 20 Jan 2017 20:32:09 +0100 Subject: [PATCH 255/257] etc/rc.d: Default to not whining about temporary jail config files being created Obtained from: ElectroBSD --- etc/defaults/rc.conf | 2 +- etc/rc.d/jail | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/defaults/rc.conf b/etc/defaults/rc.conf index c1494f933edf..ba318106d88d 100755 --- a/etc/defaults/rc.conf +++ b/etc/defaults/rc.conf @@ -701,7 +701,7 @@ iovctl_files="" # Config files for iovctl(8) ### Jail Configuration (see rc.conf(5) manual page) ########## ############################################################## jail_enable="NO" # Set to NO to disable starting of any jails -jail_confwarn="YES" # Prevent warning about obsolete per-jail configuration +jail_confwarn="NO" # Prevent warning about obsolete per-jail configuration jail_parallel_start="NO" # Start jails in the background jail_list="" # Space separated list of names of jails jail_reverse_stop="NO" # Stop jails in reverse order diff --git a/etc/rc.d/jail b/etc/rc.d/jail index f9edc1a1bd79..c408328ed9c4 100755 --- a/etc/rc.d/jail +++ b/etc/rc.d/jail @@ -146,7 +146,7 @@ parse_options() # # To relieve confusion, show a warning message. # - : ${jail_confwarn:=YES} + : ${jail_confwarn:=NO} checkyesno jail_confwarn && _confwarn=1 if [ -r "$jail_conf" -o -r "$_jconf" ]; then if ! checkyesno jail_parallel_start; then -- 2.11.0 From 6cfa243f15166ec620583e0c9f102c9c100d2bce Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 21 Jan 2017 11:02:50 +0100 Subject: [PATCH 256/257] etc/rc.d/random: Use appropriate umask in save_dev_random() Obtained from: Lu Tung-Pin who reported it on freebsd-current@freebsd.org --- etc/rc.d/random | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/rc.d/random b/etc/rc.d/random index 1e4e5b2fc729..d263877dd36d 100755 --- a/etc/rc.d/random +++ b/etc/rc.d/random @@ -20,12 +20,15 @@ saveseed_cmd="${name}_stop" save_dev_random() { + oumask=`umask` + umask 077 for f ; do if :>>"$f" ; then debug "saving entropy to $f" dd if=/dev/random of="$f" bs=4096 count=1 2>/dev/null fi done + umask ${oumask} } feed_dev_random() -- 2.11.0 From 85162884f0c25221d7cb103978a8a6fd3cd5e1f9 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 24 Jan 2017 11:34:05 +0100 Subject: [PATCH 257/257] Add reproduce.conf for ElectroBSD-r312620-6cfa243f1516 Expected hashes for ElectroBSD/amd64 (when building with itself in bhyve): SHA256 (ElectroBSD-r312620-6cfa243f1516.img) = 5aed1696f96476dad9fbd978eac3be6282f66a8c7666a934752aa9c702d8a61b SHA256 (MANIFEST) = 5cbe7802ccc72156c24439d8c6c8e8e333c4349d5be686cb34114c24330d24d9 SHA256 (base-dbg.txz) = fede9ac6cdc32ea8c82348817340253d48b8c56b8b6d39dc5274d734a0a058a4 SHA256 (base.txz) = 5c9db6230d50d20163a78aed217dca3fe7f17a78ea223d39e7c5b1222e4e3ad0 SHA256 (kernel-dbg.txz) = e00b2a689eceb60c7eb7f874c313f8154fb64803d477af3573455d3bf6e1373d SHA256 (kernel.txz) = b33563b76ad4307d7b8d440837b360e255ccc1f0687a0cdbbcc4a489a956dda6 SHA256 (src.txz) = a73b18ce45df4dc8a3e8bf225b85357fba6365796698b266cd62a260a6d6b4c5 SHA256 (tests.txz) = f6f97443152040801453dbc292271bc8f6d182e43eb7adc37b80698fe622ee31 Note that the memstick image checksum depends on external factors that haven't been completely analysed yet. While it's clear that reproducing it requires UFS as file system for /usr/obj (to let makefs get deterministic file order), this does not seem to be sufficient. The distfiles should always build reproducible, though, and the potential memstick variations only differ in a couple of bytes. The "partial image checksum" produced by release/scripts/image-checksum.sh is not affected. The expected verbose output of image-checksum.sh is: gpart checksum: 5b362b3c6728eee6af705e7fb0ffecd62ac225ffbcaeafceb970d9f78cccbe41 Boot code checksum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 mtree checksum: 626fe1eedd78fe55a7f67cb35b47038e0626d029d5045dbc0f0aa1cb32731151 Partial image checksum for ElectroBSD-r312620-6cfa243f1516.img: 3733e14af00147112e3dd0102809904a420f8d1e0646a3af463ca1af21af66eb Expected hashes for ElectroBSD/i386 (when building with itself in bhyve): SHA256 (ElectroBSD-r312620-6cfa243f1516.img) = f4a56f27194dc2c29a35eb9a4978d0c1ea1b821f1b4ade311596959b4a0e4a03 SHA256 (MANIFEST) = 23ea3b445d9fe3b31e598c6741225669911561bea178ce872ac75b07269842fa SHA256 (base-dbg.txz) = 13f92be5c628c12cb8d0dd8a634f147961693d3969014d66bded9570adb5a0a2 SHA256 (base.txz) = ae0cd7cd194be33d6f4ad63f51ae1dfc3066481fd90a8af7f23dc428bf76c5c7 SHA256 (kernel-dbg.txz) = 780d2bdb17e8238e5b54b8da76fc2c108ee0486ef4d9a8c8fabef6c9f2b55189 SHA256 (kernel.txz) = 358f059220c3f1fa489d285f8d1d09a2eae81f41d43a189b44fe597128c0c15f SHA256 (src.txz) = a73b18ce45df4dc8a3e8bf225b85357fba6365796698b266cd62a260a6d6b4c5 SHA256 (tests.txz) = 9b1a1ecd51d4222cf5a55bacc0778554febe985711d76de2670284a7cddde4c6 The expected verbose output of image-checksum.sh is: gpart checksum: 302b96e46ec4caac5b6487c5f585bcb4c8e213b2c3dbfb4d2423ab6468c6927d Boot code checksum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 mtree checksum: 4ee5136d1b0073301f032921ff2244325e29ee98c4ef5fb0e4b235809d9661b5 Partial image checksum for ElectroBSD-r312620-6cfa243f1516.img: 6124262ef375d910ef0bec9f0bd9bea7d5dbedc71ecd96b77137653b91b20e59 Obtained from: ElectroBSD --- reproduce.conf | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 reproduce.conf diff --git a/reproduce.conf b/reproduce.conf new file mode 100644 index 000000000000..0adc726b260e --- /dev/null +++ b/reproduce.conf @@ -0,0 +1,2 @@ +BUILD=ElectroBSD-r312620-6cfa243f1516 +EPOCH=1485091279 -- 2.11.0